riz/mcp-memory
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace Ollama/Windows OCR with ocrs, add clipboard image deduplication

Browse Source
This commit is contained in:
Riz Ashraf committed 2026-10-08 21:48:12 +01:00
1 parent 548f0a06ea
commit 5c5aead1a5
3 files changed
+263 -83
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/handlers/vision.rs
+117 -83
View File
@@ -289,32 +289,70 @@ pub fn get_windows_html_clipboard() -> Option<String> {
}
None
}
use ocrs::{OcrEngine, OcrEngineParams};
use std::sync::OnceLock;
#[cfg(target_os = "windows")]
pub fn perform_windows_ocr(img: &image::DynamicImage) -> Option<String> {
use windows::Graphics::Imaging::BitmapDecoder;
use windows::Media::Ocr::OcrEngine;
use windows::Storage::Streams::{DataWriter, InMemoryRandomAccessStream};
static OCR_ENGINE: OnceLock<OcrEngine> = OnceLock::new();
let mut png_bytes = std::io::Cursor::new(Vec::new());
img.write_to(&mut png_bytes, image::ImageFormat::Png).ok()?;
let bytes = png_bytes.into_inner();
pub async fn ensure_ocr_engine() -> Option<&'static OcrEngine> {
if let Some(engine) = OCR_ENGINE.get() {
return Some(engine);
}
let cache_dir = dirs::home_dir()?.join(".gemini/mcp_memory/models");
let _ = std::fs::create_dir_all(&cache_dir);
let det_path = cache_dir.join("text-detection.rten");
if !det_path.exists() || std::fs::metadata(&det_path).map(|m| m.len()).unwrap_or(0) < 100_000 {
if let Ok(resp) = reqwest::get("https://huggingface.co/robertknight/ocrs/resolve/main/text-detection.rten").await {
if let Ok(bytes) = resp.bytes().await {
let _ = std::fs::write(&det_path, bytes);
}
}
}
let rec_path = cache_dir.join("text-recognition.rten");
if !rec_path.exists() || std::fs::metadata(&rec_path).map(|m| m.len()).unwrap_or(0) < 100_000 {
if let Ok(resp) = reqwest::get("https://huggingface.co/robertknight/ocrs/resolve/main/text-recognition.rten").await {
if let Ok(bytes) = resp.bytes().await {
let _ = std::fs::write(&rec_path, bytes);
}
}
}
let stream = InMemoryRandomAccessStream::new().ok()?;
let writer = DataWriter::CreateDataWriter(&stream).ok()?;
writer.WriteBytes(&bytes).ok()?;
writer.StoreAsync().ok()?.join().ok()?;
writer.FlushAsync().ok()?.join().ok()?;
stream.Seek(0).ok()?;
let mut params = OcrEngineParams::default();
let det_model = rten::Model::load_file(&det_path).ok()?;
let rec_model = rten::Model::load_file(&rec_path).ok()?;
params.detection_model = Some(det_model);
params.recognition_model = Some(rec_model);
let decoder = BitmapDecoder::CreateAsync(&stream).ok()?.join().ok()?;
let software_bitmap = decoder.GetSoftwareBitmapAsync().ok()?.join().ok()?;
if let Ok(engine) = OcrEngine::new(params) {
let _ = OCR_ENGINE.set(engine);
OCR_ENGINE.get()
} else {
None
}
}
let engine = OcrEngine::TryCreateFromUserProfileLanguages().ok()?;
let result = engine.RecognizeAsync(&software_bitmap).ok()?.join().ok()?;
let text = result.Text().ok()?;
let s = text.to_string();
let trimmed = s.trim();
pub fn perform_ocrs_ocr(engine: &OcrEngine, img: &image::DynamicImage) -> Option<String> {
let rgb_img = img.to_rgb8();
let img_source = ocrs::ImageSource::from_bytes(
rgb_img.as_raw(),
rgb_img.dimensions(),
).ok()?;
let ocr_input = engine.prepare_input(img_source).ok()?;
let word_rects = engine.detect_words(&ocr_input).ok()?;
let line_rects = engine.find_text_lines(&ocr_input, &word_rects);
let texts = engine.recognize_text(&ocr_input, &line_rects).ok()?;
let mut out = String::new();
for text in texts.into_iter().flatten() {
out.push_str(&text.to_string());
out.push('\n');
}
let trimmed = out.trim();
if trimmed.is_empty() {
None
} else {
@@ -426,50 +464,56 @@ pub async fn process_clipboard_snapshot(state: Arc<MemoryState>) {
let path_str = file_path.to_string_lossy().to_string();
let wsl_path = to_wsl_path(&path_str);
// Deterministic OCR
let img_clone = img.clone();
let ocr_text_opt: Option<String> = tokio::task::spawn_blocking(move || {
#[cfg(target_os = "windows")]
{
perform_windows_ocr(&img_clone)
}
#[cfg(not(target_os = "windows"))]
{
None::<String>
}
})
.await
.ok()
.flatten();
let mut ocr_text_opt: Option<String> = None;
// Index OCR text into Tantivy for omni_search
if let Some(ref text) = ocr_text_opt {
let doc_id = format!("clipboard_{}", now_ms);
let _ = state
.search_index
.read()
.await
.index_clipboard_image(&doc_id, &path_str, text);
if let Some(engine) = ensure_ocr_engine().await {
let img_clone = img.clone();
ocr_text_opt = tokio::task::spawn_blocking(move || {
perform_ocrs_ocr(engine, &img_clone)
})
.await
.ok()
.flatten();
}
let cached = crate::state::CachedClipboardImage {
file_path: path_str,
file_path_wsl: wsl_path,
captured_at_epoch_ms: now_ms,
age: "0s ago".to_string(),
width: w,
height: h,
size_bytes,
ocr_text: ocr_text_opt,
};
let mut cache = state.clipboard_cache.write().await;
cache.last_image = Some(cached.clone());
cache
.history
.push_front(crate::state::ClipboardHistoryItem::Image(cached));
if cache.history.len() > 20 {
cache.history.pop_back();
// Check if this is an identical image captured rapidly
let is_duplicate = if let Some(ref prev) = cache.last_image {
prev.width == w && prev.height == h && prev.size_bytes == size_bytes
} else {
false
};
if !is_duplicate {
// Index OCR text into Tantivy for omni_search
if let Some(ref text) = ocr_text_opt {
let doc_id = format!("clipboard_{}", now_ms);
let _ = state
.search_index
.read()
.await
.index_clipboard_image(&doc_id, &path_str, text);
}
let cached = crate::state::CachedClipboardImage {
file_path: path_str,
file_path_wsl: wsl_path,
captured_at_epoch_ms: now_ms,
age: "0s ago".to_string(),
width: w,
height: h,
size_bytes,
ocr_text: ocr_text_opt,
};
cache.last_image = Some(cached.clone());
cache
.history
.push_front(crate::state::ClipboardHistoryItem::Image(cached));
if cache.history.len() > 20 {
cache.history.pop_back();
}
}
return;
}
@@ -783,17 +827,18 @@ impl McpTool for ClipboardHandler {
.to_string())
}
ClipboardAction::Read => {
let (mut out, b64_opt) =
tokio::task::spawn_blocking(move || -> crate::error::Result<(serde_json::Map<String, Value>, Option<String>)> {
let engine = ensure_ocr_engine().await;
let out =
tokio::task::spawn_blocking(move || -> crate::error::Result<serde_json::Map<String, Value>> {
let mut out = serde_json::Map::new();
let mut b64_opt = None;
if let Some(text) = get_native_clipboard_text() {
out.insert("text".into(), json!(text));
}
if let Some(dynamic_img) = get_native_clipboard_image() {
let mut img = dynamic_img;
let mut img = dynamic_img.clone();
let max_dim = 1440;
if img.width() > max_dim || img.height() > max_dim {
img = img.resize(max_dim, max_dim, FilterType::Lanczos3);
@@ -804,9 +849,6 @@ impl McpTool for ClipboardHandler {
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, 88);
if encoder.encode(&rgb_img, rgb_img.width(), rgb_img.height(), image::ExtendedColorType::Rgb8).is_ok() {
let bytes = jpeg_bytes.into_inner();
use base64::Engine;
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
b64_opt = Some(b64);
let cache_dir = dirs::home_dir()
.unwrap_or_default()
@@ -820,26 +862,18 @@ impl McpTool for ClipboardHandler {
out.insert("image_path_wsl".into(), json!(wsl_path));
}
}
if let Some(eng) = engine {
if let Some(ocr_text) = perform_ocrs_ocr(eng, &dynamic_img) {
out.insert("image_analysis".to_string(), json!(ocr_text.trim()));
}
}
}
Ok((out, b64_opt))
Ok(out)
})
.await
.map_err(|e| crate::error::AppError::Internal(format!("Task panic: {}", e)))??;
if let Some(b64_str) = b64_opt {
if state.ollama.is_available().await
&& let Ok(analysis) = state
.ollama
.generate_vision(
"Describe the content, text, or UI layout of this image concisely:",
&b64_str,
)
.await
{
out.insert("image_analysis".to_string(), json!(analysis.trim()));
}
}
state.record_activity("clipboard", "Read contents from OS clipboard", None);
Ok::<String, crate::error::AppError>(serde_json::to_string_pretty(&Value::Object(
out,