1204 lines
46 KiB
Rust
1204 lines
46 KiB
Rust
use crate::router::McpTool;
|
|
use crate::state::MemoryState;
|
|
use crate::tools::{ClipboardAction, ClipboardTool};
|
|
use arboard::{Clipboard, ImageData};
|
|
use async_trait::async_trait;
|
|
use image::{ImageBuffer, imageops::FilterType};
|
|
use serde_json::{Value, json};
|
|
use std::borrow::Cow;
|
|
use std::sync::Arc;
|
|
|
|
static CLIPBOARD_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
|
|
|
pub fn to_wsl_path(win_path: &str) -> String {
|
|
let p = win_path.trim();
|
|
if let Some(rest) = p.strip_prefix("C:\\").or_else(|| p.strip_prefix("c:\\")) {
|
|
format!("/mnt/c/{}", rest.replace('\\', "/"))
|
|
} else if let Some(rest) = p.strip_prefix("D:\\").or_else(|| p.strip_prefix("d:\\")) {
|
|
format!("/mnt/d/{}", rest.replace('\\', "/"))
|
|
} else {
|
|
p.replace('\\', "/")
|
|
}
|
|
}
|
|
|
|
pub fn extract_html_fragment(raw: &str) -> &str {
|
|
if let Some(start_idx) = raw.find("<!--StartFragment-->") {
|
|
let after_start = start_idx + "<!--StartFragment-->".len();
|
|
if let Some(end_idx) = raw[after_start..].find("<!--EndFragment-->") {
|
|
return &raw[after_start..after_start + end_idx];
|
|
}
|
|
return &raw[after_start..];
|
|
}
|
|
raw
|
|
}
|
|
|
|
pub fn html_to_markdown(html: &str) -> String {
|
|
let mut out = String::new();
|
|
let mut in_table = false;
|
|
let mut table_rows: Vec<Vec<String>> = Vec::new();
|
|
let mut current_row: Vec<String> = Vec::new();
|
|
let mut current_cell = String::new();
|
|
let mut in_cell = false;
|
|
|
|
let mut chars = html.chars().peekable();
|
|
while let Some(c) = chars.next() {
|
|
if c == '<' {
|
|
let mut tag = String::new();
|
|
while let Some(&next_c) = chars.peek() {
|
|
chars.next();
|
|
if next_c == '>' {
|
|
break;
|
|
}
|
|
tag.push(next_c);
|
|
}
|
|
let tag_lower = tag.trim().to_lowercase();
|
|
let tag_name = tag_lower.split_whitespace().next().unwrap_or("");
|
|
|
|
match tag_name {
|
|
"table" => {
|
|
in_table = true;
|
|
table_rows.clear();
|
|
}
|
|
"/table" => {
|
|
if in_table {
|
|
in_table = false;
|
|
if !table_rows.is_empty() {
|
|
out.push('\n');
|
|
let num_cols = table_rows.iter().map(|r| r.len()).max().unwrap_or(0);
|
|
for (row_idx, row) in table_rows.iter().enumerate() {
|
|
out.push('|');
|
|
for col_idx in 0..num_cols {
|
|
let cell_val = row.get(col_idx).map(|s| s.trim()).unwrap_or("");
|
|
out.push(' ');
|
|
out.push_str(cell_val);
|
|
out.push_str(" |");
|
|
}
|
|
out.push('\n');
|
|
if row_idx == 0 {
|
|
out.push('|');
|
|
for _ in 0..num_cols {
|
|
out.push_str(" --- |");
|
|
}
|
|
out.push('\n');
|
|
}
|
|
}
|
|
out.push('\n');
|
|
}
|
|
}
|
|
}
|
|
"tr" => {
|
|
if in_table {
|
|
current_row.clear();
|
|
}
|
|
}
|
|
"/tr" => {
|
|
if in_table {
|
|
if !current_row.is_empty() {
|
|
table_rows.push(current_row.clone());
|
|
}
|
|
current_row.clear();
|
|
}
|
|
}
|
|
"th" | "td" => {
|
|
if in_table {
|
|
in_cell = true;
|
|
current_cell.clear();
|
|
}
|
|
}
|
|
"/th" | "/td" => {
|
|
if in_table && in_cell {
|
|
in_cell = false;
|
|
current_row.push(current_cell.trim().to_string());
|
|
current_cell.clear();
|
|
}
|
|
}
|
|
"br" | "br/" => {
|
|
if in_cell {
|
|
current_cell.push(' ');
|
|
} else {
|
|
out.push('\n');
|
|
}
|
|
}
|
|
"p" | "div" => {
|
|
if !in_cell && !out.ends_with("\n\n") && !out.is_empty() {
|
|
out.push('\n');
|
|
}
|
|
}
|
|
"/p" | "/div" => {
|
|
if !in_cell && !out.ends_with("\n\n") {
|
|
out.push('\n');
|
|
}
|
|
}
|
|
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
|
|
if !in_cell {
|
|
if !out.ends_with('\n') && !out.is_empty() {
|
|
out.push('\n');
|
|
}
|
|
let level = tag_name[1..].parse::<usize>().unwrap_or(1);
|
|
out.push_str(&"#".repeat(level));
|
|
out.push(' ');
|
|
}
|
|
}
|
|
"/h1" | "/h2" | "/h3" | "/h4" | "/h5" | "/h6" => {
|
|
if !in_cell {
|
|
out.push('\n');
|
|
}
|
|
}
|
|
"b" | "strong" => {
|
|
if in_cell {
|
|
current_cell.push_str("**");
|
|
} else {
|
|
out.push_str("**");
|
|
}
|
|
}
|
|
"/b" | "/strong" => {
|
|
if in_cell {
|
|
current_cell.push_str("**");
|
|
} else {
|
|
out.push_str("**");
|
|
}
|
|
}
|
|
"i" | "em" => {
|
|
if in_cell {
|
|
current_cell.push('*');
|
|
} else {
|
|
out.push('*');
|
|
}
|
|
}
|
|
"/i" | "/em" => {
|
|
if in_cell {
|
|
current_cell.push('*');
|
|
} else {
|
|
out.push('*');
|
|
}
|
|
}
|
|
"code" => {
|
|
if in_cell {
|
|
current_cell.push('`');
|
|
} else {
|
|
out.push('`');
|
|
}
|
|
}
|
|
"/code" => {
|
|
if in_cell {
|
|
current_cell.push('`');
|
|
} else {
|
|
out.push('`');
|
|
}
|
|
}
|
|
"li" => {
|
|
if !in_cell {
|
|
out.push_str("* ");
|
|
}
|
|
}
|
|
"/li"
|
|
if !in_cell => {
|
|
out.push('\n');
|
|
}
|
|
_ => {}
|
|
}
|
|
} else if c == '&' {
|
|
let mut entity = String::new();
|
|
while let Some(&next_c) = chars.peek() {
|
|
chars.next();
|
|
if next_c == ';' {
|
|
break;
|
|
}
|
|
entity.push(next_c);
|
|
}
|
|
let decoded = match entity.as_str() {
|
|
"nbsp" => " ",
|
|
"amp" => "&",
|
|
"lt" => "<",
|
|
"gt" => ">",
|
|
"quot" => "\"",
|
|
"apos" | "#39" => "'",
|
|
_ => " ",
|
|
};
|
|
if in_cell {
|
|
current_cell.push_str(decoded);
|
|
} else if !in_table {
|
|
out.push_str(decoded);
|
|
}
|
|
} else if in_cell {
|
|
current_cell.push(c);
|
|
} else if !in_table {
|
|
out.push(c);
|
|
}
|
|
}
|
|
|
|
out.trim().to_string()
|
|
}
|
|
|
|
pub fn get_native_clipboard_text() -> Option<String> {
|
|
let _guard = CLIPBOARD_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
for _ in 0..3 {
|
|
if let Ok(mut clipboard) = arboard::Clipboard::new()
|
|
&& let Ok(text) = clipboard.get_text()
|
|
&& !text.trim().is_empty()
|
|
{
|
|
return Some(text);
|
|
}
|
|
std::thread::sleep(std::time::Duration::from_millis(20));
|
|
}
|
|
None
|
|
}
|
|
|
|
#[cfg(target_os = "windows")]
|
|
pub fn get_windows_html_clipboard() -> Option<String> {
|
|
use windows_sys::Win32::System::DataExchange::{
|
|
CloseClipboard, GetClipboardData, IsClipboardFormatAvailable, OpenClipboard,
|
|
RegisterClipboardFormatW,
|
|
};
|
|
use windows_sys::Win32::System::Memory::{GlobalLock, GlobalSize, GlobalUnlock};
|
|
let format_name: Vec<u16> = "HTML Format\0".encode_utf16().collect();
|
|
unsafe {
|
|
let format_id = RegisterClipboardFormatW(format_name.as_ptr());
|
|
if format_id == 0 {
|
|
return None;
|
|
}
|
|
for _ in 0..3 {
|
|
if OpenClipboard(std::ptr::null_mut()) != 0 {
|
|
let has_format = IsClipboardFormatAvailable(format_id) != 0;
|
|
let mut html_opt = None;
|
|
if has_format {
|
|
let handle = GetClipboardData(format_id);
|
|
if !handle.is_null() {
|
|
let size = GlobalSize(handle as _);
|
|
if size > 0 {
|
|
let ptr = GlobalLock(handle as _);
|
|
if !ptr.is_null() {
|
|
let slice = std::slice::from_raw_parts(ptr as *const u8, size);
|
|
if let Ok(s) = std::str::from_utf8(slice) {
|
|
html_opt = Some(s.to_string());
|
|
}
|
|
GlobalUnlock(handle as _);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
CloseClipboard();
|
|
if html_opt.is_some() {
|
|
return html_opt;
|
|
}
|
|
break;
|
|
}
|
|
std::thread::sleep(std::time::Duration::from_millis(20));
|
|
}
|
|
}
|
|
None
|
|
}
|
|
use ocrs::{OcrEngine, OcrEngineParams};
|
|
use std::sync::OnceLock;
|
|
|
|
static OCR_ENGINE: OnceLock<OcrEngine> = OnceLock::new();
|
|
|
|
pub async fn ensure_ocr_engine() -> Option<&'static OcrEngine> {
|
|
if let Some(engine) = OCR_ENGINE.get() {
|
|
return Some(engine);
|
|
}
|
|
|
|
let cache_dir = dirs::home_dir()?.join(".gemini/mcp_memory/models");
|
|
let _ = std::fs::create_dir_all(&cache_dir);
|
|
|
|
let det_path = cache_dir.join("text-detection.rten");
|
|
if !det_path.exists() || std::fs::metadata(&det_path).map(|m| m.len()).unwrap_or(0) < 100_000 {
|
|
match reqwest::get("https://ocrs-models.s3-accelerate.amazonaws.com/text-detection.rten")
|
|
.await
|
|
{
|
|
Ok(resp) => {
|
|
if let Ok(bytes) = resp.bytes().await {
|
|
if let Err(e) = std::fs::write(&det_path, &bytes) {
|
|
tracing::error!("Failed to write text-detection.rten to disk: {}", e);
|
|
}
|
|
} else {
|
|
tracing::error!(
|
|
"Failed to read bytes from text-detection.rten download response"
|
|
);
|
|
}
|
|
}
|
|
Err(e) => tracing::error!("Failed to download text-detection.rten: {}", e),
|
|
}
|
|
}
|
|
|
|
let rec_path = cache_dir.join("text-recognition.rten");
|
|
if !rec_path.exists() || std::fs::metadata(&rec_path).map(|m| m.len()).unwrap_or(0) < 100_000 {
|
|
match reqwest::get("https://ocrs-models.s3-accelerate.amazonaws.com/text-recognition.rten")
|
|
.await
|
|
{
|
|
Ok(resp) => {
|
|
if let Ok(bytes) = resp.bytes().await {
|
|
if let Err(e) = std::fs::write(&rec_path, &bytes) {
|
|
tracing::error!("Failed to write text-recognition.rten to disk: {}", e);
|
|
}
|
|
} else {
|
|
tracing::error!(
|
|
"Failed to read bytes from text-recognition.rten download response"
|
|
);
|
|
}
|
|
}
|
|
Err(e) => tracing::error!("Failed to download text-recognition.rten: {}", e),
|
|
}
|
|
}
|
|
|
|
let mut params = OcrEngineParams::default();
|
|
let det_model = rten::Model::load_file(&det_path).ok()?;
|
|
let rec_model = rten::Model::load_file(&rec_path).ok()?;
|
|
params.detection_model = Some(det_model);
|
|
params.recognition_model = Some(rec_model);
|
|
|
|
if let Ok(engine) = OcrEngine::new(params) {
|
|
let _ = OCR_ENGINE.set(engine);
|
|
OCR_ENGINE.get()
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
pub fn perform_ocrs_ocr(engine: &OcrEngine, img: &image::DynamicImage) -> Option<String> {
|
|
let rgb_img = img.to_rgb8();
|
|
let img_source = ocrs::ImageSource::from_bytes(rgb_img.as_raw(), rgb_img.dimensions()).ok()?;
|
|
|
|
let ocr_input = engine.prepare_input(img_source).ok()?;
|
|
let word_rects = engine.detect_words(&ocr_input).ok()?;
|
|
let line_rects = engine.find_text_lines(&ocr_input, &word_rects);
|
|
let texts = engine.recognize_text(&ocr_input, &line_rects).ok()?;
|
|
|
|
let mut out = String::new();
|
|
for text in texts.into_iter().flatten() {
|
|
out.push_str(&text.to_string());
|
|
out.push('\n');
|
|
}
|
|
|
|
let trimmed = out.trim();
|
|
if trimmed.is_empty() {
|
|
None
|
|
} else {
|
|
Some(trimmed.to_string())
|
|
}
|
|
}
|
|
|
|
#[cfg(target_os = "windows")]
|
|
fn get_windows_png_clipboard_image() -> Option<image::DynamicImage> {
|
|
use windows_sys::Win32::System::DataExchange::{
|
|
CloseClipboard, GetClipboardData, IsClipboardFormatAvailable, OpenClipboard,
|
|
RegisterClipboardFormatW,
|
|
};
|
|
use windows_sys::Win32::System::Memory::{GlobalLock, GlobalSize, GlobalUnlock};
|
|
let format_name: Vec<u16> = "PNG\0".encode_utf16().collect();
|
|
unsafe {
|
|
let format_id = RegisterClipboardFormatW(format_name.as_ptr());
|
|
if format_id == 0 {
|
|
return None;
|
|
}
|
|
for _ in 0..3 {
|
|
if OpenClipboard(std::ptr::null_mut()) != 0 {
|
|
let has_format = IsClipboardFormatAvailable(format_id) != 0;
|
|
let mut img_opt = None;
|
|
if has_format {
|
|
let handle = GetClipboardData(format_id);
|
|
if !handle.is_null() {
|
|
let size = GlobalSize(handle as _);
|
|
if size > 0 {
|
|
let ptr = GlobalLock(handle as _);
|
|
if !ptr.is_null() {
|
|
let slice = std::slice::from_raw_parts(ptr as *const u8, size);
|
|
img_opt = image::load_from_memory(slice).ok();
|
|
GlobalUnlock(handle as _);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
CloseClipboard();
|
|
if img_opt.is_some() {
|
|
return img_opt;
|
|
}
|
|
break;
|
|
}
|
|
std::thread::sleep(std::time::Duration::from_millis(20));
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
pub fn get_native_clipboard_image() -> Option<image::DynamicImage> {
|
|
let _guard = CLIPBOARD_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
#[cfg(target_os = "windows")]
|
|
if let Some(img) = get_windows_png_clipboard_image() {
|
|
return Some(img);
|
|
}
|
|
for _ in 0..3 {
|
|
if let Ok(mut clipboard) = arboard::Clipboard::new()
|
|
&& let Ok(image_data) = clipboard.get_image()
|
|
&& let Some(img) = ImageBuffer::<image::Rgba<u8>, _>::from_raw(
|
|
image_data.width as u32,
|
|
image_data.height as u32,
|
|
image_data.bytes.into_owned(),
|
|
)
|
|
{
|
|
return Some(image::DynamicImage::ImageRgba8(img));
|
|
}
|
|
std::thread::sleep(std::time::Duration::from_millis(20));
|
|
}
|
|
None
|
|
}
|
|
|
|
pub async fn process_clipboard_snapshot(state: Arc<MemoryState>) {
|
|
let now_ms = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_millis() as u64;
|
|
|
|
// 1. Try to read image first
|
|
let img_opt = tokio::task::spawn_blocking(get_native_clipboard_image)
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
if let Some(mut img) = img_opt {
|
|
let max_dim = 1440;
|
|
if img.width() > max_dim || img.height() > max_dim {
|
|
img = img.resize(max_dim, max_dim, FilterType::Lanczos3);
|
|
}
|
|
let (w, h) = (img.width(), img.height());
|
|
let rgb_img = img.to_rgb8();
|
|
|
|
let mut jpeg_bytes = std::io::Cursor::new(Vec::new());
|
|
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, 88);
|
|
if encoder
|
|
.encode(&rgb_img, w, h, image::ExtendedColorType::Rgb8)
|
|
.is_ok()
|
|
{
|
|
let bytes = jpeg_bytes.into_inner();
|
|
let size_bytes = bytes.len();
|
|
|
|
let cache_dir = dirs::home_dir()
|
|
.unwrap_or_default()
|
|
.join(".gemini/mcp_memory/clipboard");
|
|
let _ = std::fs::create_dir_all(&cache_dir);
|
|
let file_path = cache_dir.join("clipboard_latest_image.jpg");
|
|
let _ = std::fs::write(&file_path, &bytes);
|
|
|
|
let path_str = file_path.to_string_lossy().to_string();
|
|
let wsl_path = to_wsl_path(&path_str);
|
|
|
|
let mut ocr_text_opt: Option<String> = None;
|
|
|
|
if let Some(engine) = ensure_ocr_engine().await {
|
|
let img_clone = img.clone();
|
|
ocr_text_opt =
|
|
tokio::task::spawn_blocking(move || perform_ocrs_ocr(engine, &img_clone))
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
}
|
|
|
|
let mut cache = state.clipboard_cache.write().await;
|
|
|
|
// Check if this is an identical image captured rapidly
|
|
let is_duplicate = if let Some(ref prev) = cache.last_image {
|
|
prev.width == w && prev.height == h && prev.size_bytes == size_bytes
|
|
} else {
|
|
false
|
|
};
|
|
|
|
if !is_duplicate {
|
|
// Index OCR text into Tantivy for omni_search
|
|
if let Some(ref text) = ocr_text_opt {
|
|
let doc_id = format!("clipboard_{}", now_ms);
|
|
let _ = state
|
|
.search_index
|
|
.read()
|
|
.await
|
|
.index_clipboard_image(&doc_id, &path_str, text);
|
|
}
|
|
|
|
let cached = crate::state::CachedClipboardImage {
|
|
file_path: path_str,
|
|
file_path_wsl: wsl_path,
|
|
captured_at_epoch_ms: now_ms,
|
|
age: "0s ago".to_string(),
|
|
width: w,
|
|
height: h,
|
|
size_bytes,
|
|
ocr_text: ocr_text_opt,
|
|
};
|
|
|
|
cache.last_image = Some(cached.clone());
|
|
cache
|
|
.history
|
|
.push_front(crate::state::ClipboardHistoryItem::Image(cached));
|
|
if cache.history.len() > 20 {
|
|
cache.history.pop_back();
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
|
|
// 2. Try text / HTML
|
|
let text_opt = tokio::task::spawn_blocking(|| {
|
|
#[cfg(target_os = "windows")]
|
|
if let Some(raw_html) = get_windows_html_clipboard() {
|
|
let frag = extract_html_fragment(&raw_html);
|
|
let md = html_to_markdown(frag);
|
|
if !md.trim().is_empty() {
|
|
return Some(md);
|
|
}
|
|
}
|
|
get_native_clipboard_text()
|
|
})
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
if let Some(text) = text_opt {
|
|
let text_trim = text.trim();
|
|
if text_trim.is_empty() {
|
|
return;
|
|
}
|
|
let char_count = text_trim.chars().count();
|
|
let cached = crate::state::CachedClipboardText {
|
|
text: text_trim.to_string(),
|
|
captured_at_epoch_ms: now_ms,
|
|
age: "0s ago".to_string(),
|
|
char_count,
|
|
};
|
|
|
|
let mut cache = state.clipboard_cache.write().await;
|
|
if let Some(ref prev) = cache.last_text
|
|
&& prev.text == cached.text {
|
|
return;
|
|
}
|
|
cache.last_text = Some(cached.clone());
|
|
cache
|
|
.history
|
|
.push_front(crate::state::ClipboardHistoryItem::Text(cached));
|
|
if cache.history.len() > 20 {
|
|
cache.history.pop_back();
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(target_os = "windows")]
|
|
static CLIPBOARD_NOTIFY_TX: std::sync::OnceLock<tokio::sync::mpsc::UnboundedSender<()>> =
|
|
std::sync::OnceLock::new();
|
|
|
|
#[cfg(target_os = "windows")]
|
|
pub fn spawn_clipboard_listener(state: Arc<MemoryState>) {
|
|
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<()>();
|
|
let _ = CLIPBOARD_NOTIFY_TX.set(tx);
|
|
|
|
std::thread::Builder::new()
|
|
.name("win32-clipboard-listener".to_string())
|
|
.spawn(move || {
|
|
use windows_sys::Win32::Foundation::*;
|
|
use windows_sys::Win32::System::DataExchange::*;
|
|
use windows_sys::Win32::UI::WindowsAndMessaging::*;
|
|
|
|
unsafe extern "system" fn wnd_proc(
|
|
hwnd: HWND,
|
|
msg: u32,
|
|
wparam: WPARAM,
|
|
lparam: LPARAM,
|
|
) -> LRESULT {
|
|
if msg == WM_CLIPBOARDUPDATE {
|
|
if let Some(tx) = CLIPBOARD_NOTIFY_TX.get() {
|
|
let _ = tx.send(());
|
|
}
|
|
return 0;
|
|
}
|
|
unsafe { DefWindowProcW(hwnd, msg, wparam, lparam) }
|
|
}
|
|
|
|
unsafe {
|
|
let class_name: Vec<u16> =
|
|
"McpMemoryClipboardWatcherClass\0".encode_utf16().collect();
|
|
let wnd_class = WNDCLASSEXW {
|
|
cbSize: std::mem::size_of::<WNDCLASSEXW>() as u32,
|
|
style: 0,
|
|
lpfnWndProc: Some(wnd_proc),
|
|
cbClsExtra: 0,
|
|
cbWndExtra: 0,
|
|
hInstance: 0 as _,
|
|
hIcon: 0 as _,
|
|
hCursor: 0 as _,
|
|
hbrBackground: 0 as _,
|
|
lpszMenuName: std::ptr::null(),
|
|
lpszClassName: class_name.as_ptr(),
|
|
hIconSm: 0 as _,
|
|
};
|
|
RegisterClassExW(&wnd_class);
|
|
|
|
let hwnd = CreateWindowExW(
|
|
0,
|
|
class_name.as_ptr(),
|
|
std::ptr::null(),
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
HWND_MESSAGE,
|
|
0 as _,
|
|
0 as _,
|
|
std::ptr::null(),
|
|
);
|
|
if hwnd == 0 as _ {
|
|
tracing::error!("Failed to create Win32 clipboard message window");
|
|
return;
|
|
}
|
|
|
|
if AddClipboardFormatListener(hwnd) == 0 {
|
|
tracing::error!("Failed to AddClipboardFormatListener");
|
|
DestroyWindow(hwnd);
|
|
return;
|
|
}
|
|
tracing::info!("Win32 clipboard listener registered successfully on HWND_MESSAGE.");
|
|
|
|
let mut msg: MSG = std::mem::zeroed();
|
|
while GetMessageW(&mut msg, 0 as _, 0, 0) > 0 {
|
|
TranslateMessage(&msg);
|
|
DispatchMessageW(&msg);
|
|
}
|
|
|
|
RemoveClipboardFormatListener(hwnd);
|
|
DestroyWindow(hwnd);
|
|
}
|
|
})
|
|
.expect("Failed to spawn win32-clipboard-listener thread");
|
|
|
|
tokio::spawn(async move {
|
|
while let Some(()) = rx.recv().await {
|
|
tokio::time::sleep(tokio::time::Duration::from_millis(150)).await;
|
|
while rx.try_recv().is_ok() {}
|
|
|
|
process_clipboard_snapshot(Arc::clone(&state)).await;
|
|
}
|
|
});
|
|
}
|
|
|
|
#[cfg(not(target_os = "windows"))]
|
|
pub fn spawn_clipboard_listener(_state: Arc<MemoryState>) {
|
|
tracing::info!("Clipboard listener: running in headless/Linux mode (no Win32 listener).");
|
|
}
|
|
|
|
pub struct ClipboardHandler;
|
|
|
|
#[async_trait]
|
|
impl McpTool for ClipboardHandler {
|
|
fn name(&self) -> &'static str {
|
|
"clipboard"
|
|
}
|
|
|
|
fn schema(&self) -> Value {
|
|
crate::mcp::tool_def::<ClipboardTool>(
|
|
"clipboard",
|
|
"Smart OS clipboard management: Retrieve screenshots ('image') with overwrite immunity and verbatim OCR, normalized text ('text'), recent history ('history'), clear, read, or write.",
|
|
)
|
|
}
|
|
|
|
async fn execute(&self, args: Value, state: Arc<MemoryState>) -> crate::error::Result<String> {
|
|
let req: ClipboardTool = serde_json::from_value(args).map_err(|e| e.to_string())?;
|
|
match req.action {
|
|
ClipboardAction::Image => {
|
|
let cached_opt = {
|
|
let cache = state.clipboard_cache.read().await;
|
|
cache.last_image.clone()
|
|
};
|
|
|
|
let item = if let Some(mut img) = cached_opt {
|
|
img.age = crate::state::format_age(img.captured_at_epoch_ms);
|
|
Some(img)
|
|
} else {
|
|
process_clipboard_snapshot(Arc::clone(&state)).await;
|
|
let cache = state.clipboard_cache.read().await;
|
|
cache.last_image.clone().map(|mut img| {
|
|
img.age = crate::state::format_age(img.captured_at_epoch_ms);
|
|
img
|
|
})
|
|
};
|
|
|
|
let item = if item.is_none() {
|
|
let cache_dir = dirs::home_dir()
|
|
.unwrap_or_default()
|
|
.join(".gemini/mcp_memory/clipboard");
|
|
let file_path = cache_dir.join("clipboard_latest_image.jpg");
|
|
if file_path.exists() {
|
|
let path_str = file_path.to_string_lossy().to_string();
|
|
let wsl_path = to_wsl_path(&path_str);
|
|
let meta = std::fs::metadata(&file_path).ok();
|
|
let size_bytes = meta.as_ref().map(|m| m.len() as usize).unwrap_or(0);
|
|
let epoch_ms = meta
|
|
.and_then(|m| m.modified().ok())
|
|
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
|
.map(|d| d.as_millis() as u64)
|
|
.unwrap_or_default();
|
|
Some(crate::state::CachedClipboardImage {
|
|
file_path: path_str,
|
|
file_path_wsl: wsl_path,
|
|
captured_at_epoch_ms: epoch_ms,
|
|
age: crate::state::format_age(epoch_ms),
|
|
width: 0,
|
|
height: 0,
|
|
size_bytes,
|
|
ocr_text: None,
|
|
})
|
|
} else {
|
|
None
|
|
}
|
|
} else {
|
|
item
|
|
};
|
|
|
|
state.record_activity(
|
|
"clipboard",
|
|
"Retrieved screenshot image from clipboard cache",
|
|
None,
|
|
);
|
|
|
|
if let Some(img) = item {
|
|
Ok(serde_yaml::to_string(&img)?)
|
|
} else {
|
|
Ok(json!({
|
|
"status": "no_image_in_clipboard",
|
|
"message": "No screenshot or image found in clipboard cache. Take a screenshot (Win+Shift+S on Windows) and call clipboard(action: \"image\") again."
|
|
}).to_string())
|
|
}
|
|
}
|
|
ClipboardAction::Text => {
|
|
let cached_opt = {
|
|
let cache = state.clipboard_cache.read().await;
|
|
cache.last_text.clone()
|
|
};
|
|
|
|
let item = if let Some(mut txt) = cached_opt {
|
|
txt.age = crate::state::format_age(txt.captured_at_epoch_ms);
|
|
Some(txt)
|
|
} else {
|
|
process_clipboard_snapshot(Arc::clone(&state)).await;
|
|
let cache = state.clipboard_cache.read().await;
|
|
cache.last_text.clone().map(|mut txt| {
|
|
txt.age = crate::state::format_age(txt.captured_at_epoch_ms);
|
|
txt
|
|
})
|
|
};
|
|
|
|
state.record_activity("clipboard", "Retrieved text from clipboard cache", None);
|
|
if let Some(txt) = item {
|
|
Ok(serde_yaml::to_string(&txt)?)
|
|
} else {
|
|
Ok(json!({
|
|
"text": "",
|
|
"status": "empty",
|
|
"message": "Clipboard does not currently contain text."
|
|
})
|
|
.to_string())
|
|
}
|
|
}
|
|
ClipboardAction::History => {
|
|
let history = {
|
|
let cache = state.clipboard_cache.read().await;
|
|
cache
|
|
.history
|
|
.iter()
|
|
.map(|item| match item {
|
|
crate::state::ClipboardHistoryItem::Image(img) => {
|
|
let mut img = img.clone();
|
|
img.age = crate::state::format_age(img.captured_at_epoch_ms);
|
|
crate::state::ClipboardHistoryItem::Image(img)
|
|
}
|
|
crate::state::ClipboardHistoryItem::Text(txt) => {
|
|
let mut txt = txt.clone();
|
|
txt.age = crate::state::format_age(txt.captured_at_epoch_ms);
|
|
crate::state::ClipboardHistoryItem::Text(txt)
|
|
}
|
|
})
|
|
.collect::<Vec<_>>()
|
|
};
|
|
|
|
state.record_activity("clipboard", "Viewed clipboard history ring buffer", None);
|
|
Ok(serde_yaml::to_string(&history)?)
|
|
}
|
|
ClipboardAction::Clear => {
|
|
tokio::task::spawn_blocking(|| {
|
|
let _guard = CLIPBOARD_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
if let Ok(mut c) = Clipboard::new() {
|
|
let _ = c.clear();
|
|
}
|
|
})
|
|
.await
|
|
.map_err(|e| crate::error::AppError::Internal(e.to_string()))?;
|
|
|
|
let mut cache = state.clipboard_cache.write().await;
|
|
cache.last_image = None;
|
|
cache.last_text = None;
|
|
cache.history.clear();
|
|
|
|
state.record_activity("clipboard", "Cleared OS clipboard and memory cache", None);
|
|
Ok(json!({
|
|
"status": "cleared",
|
|
"message": "OS clipboard and memory cache successfully cleared."
|
|
})
|
|
.to_string())
|
|
}
|
|
ClipboardAction::Read => {
|
|
let engine = ensure_ocr_engine().await;
|
|
|
|
let out = tokio::task::spawn_blocking(
|
|
move || -> crate::error::Result<serde_json::Map<String, Value>> {
|
|
let mut out = serde_json::Map::new();
|
|
|
|
if let Some(text) = get_native_clipboard_text() {
|
|
out.insert("text".into(), json!(text));
|
|
}
|
|
|
|
if let Some(dynamic_img) = get_native_clipboard_image() {
|
|
let mut img = dynamic_img.clone();
|
|
let max_dim = 1440;
|
|
if img.width() > max_dim || img.height() > max_dim {
|
|
img = img.resize(max_dim, max_dim, FilterType::Lanczos3);
|
|
}
|
|
let rgb_img = img.into_rgb8();
|
|
|
|
let mut jpeg_bytes = std::io::Cursor::new(Vec::new());
|
|
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(
|
|
&mut jpeg_bytes,
|
|
88,
|
|
);
|
|
if encoder
|
|
.encode(
|
|
&rgb_img,
|
|
rgb_img.width(),
|
|
rgb_img.height(),
|
|
image::ExtendedColorType::Rgb8,
|
|
)
|
|
.is_ok()
|
|
{
|
|
let bytes = jpeg_bytes.into_inner();
|
|
|
|
let cache_dir = dirs::home_dir()
|
|
.unwrap_or_default()
|
|
.join(".gemini/mcp_memory/clipboard");
|
|
let _ = std::fs::create_dir_all(&cache_dir);
|
|
let file_path = cache_dir.join("clipboard_latest_image.jpg");
|
|
if std::fs::write(&file_path, &bytes).is_ok() {
|
|
let path_str = file_path.to_string_lossy().to_string();
|
|
out.insert("image_path".into(), json!(path_str));
|
|
let wsl_path = to_wsl_path(&path_str);
|
|
out.insert("image_path_wsl".into(), json!(wsl_path));
|
|
}
|
|
}
|
|
|
|
if let Some(eng) = engine
|
|
&& let Some(ocr_text) = perform_ocrs_ocr(eng, &dynamic_img) {
|
|
out.insert(
|
|
"image_analysis".to_string(),
|
|
json!(ocr_text.trim()),
|
|
);
|
|
}
|
|
}
|
|
Ok(out)
|
|
},
|
|
)
|
|
.await
|
|
.map_err(|e| crate::error::AppError::Internal(format!("Task panic: {}", e)))??;
|
|
|
|
state.record_activity("clipboard", "Read contents from OS clipboard", None);
|
|
Ok::<String, crate::error::AppError>(serde_yaml::to_string(&Value::Object(
|
|
out,
|
|
))?)
|
|
}
|
|
ClipboardAction::Write => {
|
|
let text_opt = req.text;
|
|
let image_path_opt = req.image_path;
|
|
let res = tokio::task::spawn_blocking(move || {
|
|
let _guard = CLIPBOARD_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
let mut msgs = Vec::new();
|
|
|
|
if let Some(text) = &text_opt {
|
|
let mut written = false;
|
|
if let Ok(mut clipboard) = Clipboard::new() {
|
|
for _ in 0..3 {
|
|
if clipboard.set_text(text).is_ok() {
|
|
written = true;
|
|
break;
|
|
}
|
|
std::thread::sleep(std::time::Duration::from_millis(20));
|
|
}
|
|
}
|
|
if written {
|
|
msgs.push("Wrote text");
|
|
}
|
|
}
|
|
|
|
if let Some(image_path) = &image_path_opt {
|
|
match image::open(image_path) {
|
|
Ok(img) => {
|
|
let img = if img.width() > 2048 || img.height() > 2048 {
|
|
img.resize(2048, 2048, FilterType::Triangle)
|
|
} else {
|
|
img
|
|
};
|
|
let rgba = img.into_rgba8();
|
|
let (w, h) = rgba.dimensions();
|
|
let raw_bytes = rgba.into_raw();
|
|
let mut written = false;
|
|
if let Ok(mut clipboard) = arboard::Clipboard::new() {
|
|
for _ in 0..3 {
|
|
let img_data = ImageData {
|
|
width: w as usize,
|
|
height: h as usize,
|
|
bytes: Cow::Borrowed(&raw_bytes),
|
|
};
|
|
if clipboard.set_image(img_data).is_ok() {
|
|
written = true;
|
|
break;
|
|
}
|
|
std::thread::sleep(std::time::Duration::from_millis(20));
|
|
}
|
|
}
|
|
if written {
|
|
msgs.push("Wrote image");
|
|
} else {
|
|
return Err(crate::error::AppError::Internal(
|
|
"Failed to write image to clipboard".to_string(),
|
|
));
|
|
}
|
|
}
|
|
Err(e) => {
|
|
return Err(crate::error::AppError::Internal(format!(
|
|
"Failed to open image {}: {}",
|
|
image_path, e
|
|
)));
|
|
}
|
|
}
|
|
}
|
|
|
|
if msgs.is_empty() {
|
|
Ok("No valid clipboard data provided or failed to write.".to_string())
|
|
} else {
|
|
Ok(format!(
|
|
"Successfully populated clipboard: {}",
|
|
msgs.join(", ")
|
|
))
|
|
}
|
|
})
|
|
.await
|
|
.map_err(|e| crate::error::AppError::Internal(format!("Task panic: {}", e)))??;
|
|
|
|
state.record_activity("clipboard", &res, None);
|
|
Ok(res)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use serde_json::json;
|
|
use std::sync::Arc;
|
|
use tempfile::tempdir;
|
|
|
|
#[tokio::test]
|
|
async fn test_write_clipboard() {
|
|
let dir = tempdir().unwrap();
|
|
let state = Arc::new(MemoryState::new(dir.path().to_str().unwrap()));
|
|
let handler = ClipboardHandler;
|
|
|
|
let args = json!({
|
|
"action": "write",
|
|
"text": "test_text"
|
|
});
|
|
|
|
let result = handler
|
|
.execute(args, state)
|
|
.await
|
|
.map_err(|e| format!("Failed to write clipboard: {}", e))
|
|
.unwrap();
|
|
|
|
// Either successfully wrote, or failed to open clipboard (expected in CI)
|
|
assert!(
|
|
result.contains("Successfully populated")
|
|
|| result.contains("No valid clipboard data")
|
|
|| result.contains("Failed to write image")
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_read_clipboard() {
|
|
let dir = tempdir().unwrap();
|
|
let state = Arc::new(MemoryState::new(dir.path().to_str().unwrap()));
|
|
let handler = ClipboardHandler;
|
|
|
|
let result = handler
|
|
.execute(json!({"action": "read"}), state)
|
|
.await
|
|
.map_err(|e| format!("Failed to read clipboard: {}", e))
|
|
.unwrap();
|
|
|
|
// Returns a JSON string, possibly {}
|
|
let parsed: serde_json::Value = serde_yaml::from_str(&result).unwrap();
|
|
assert!(parsed.is_object());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_read_clipboard_empty() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let state = Arc::new(MemoryState::new(dir.path().to_str().unwrap()));
|
|
let handler = ClipboardHandler;
|
|
|
|
let result = handler
|
|
.execute(serde_json::json!({"action": "read"}), state)
|
|
.await
|
|
.map_err(|e| format!("Failed to read clipboard: {}", e))
|
|
.unwrap();
|
|
|
|
let parsed: serde_json::Value = serde_yaml::from_str(&result).unwrap();
|
|
assert!(parsed.is_object());
|
|
}
|
|
|
|
#[test]
|
|
fn test_no_subprocess_clipboard_regression() {
|
|
let vision_src = include_str!("vision.rs");
|
|
let code_only = vision_src.split("mod tests").next().unwrap_or(vision_src);
|
|
let forbidden_cmd = format!("Command::{}{}", "n", "ew");
|
|
let forbidden_ps = format!("power{}", "shell");
|
|
let forbidden_wl = format!("wl-{}", "paste");
|
|
let forbidden_xc = format!("x{}", "clip");
|
|
|
|
assert!(
|
|
!code_only.contains(&forbidden_cmd),
|
|
"Regression detected: vision.rs must not spawn subprocesses!"
|
|
);
|
|
assert!(
|
|
!code_only.contains(&forbidden_ps),
|
|
"Regression detected: vision.rs must not invoke powershell!"
|
|
);
|
|
assert!(
|
|
!code_only.contains(&forbidden_wl),
|
|
"Regression detected: vision.rs must not invoke wl-paste!"
|
|
);
|
|
assert!(
|
|
!code_only.contains(&forbidden_xc),
|
|
"Regression detected: vision.rs must not invoke xclip!"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_wsl_path_translation() {
|
|
assert_eq!(
|
|
to_wsl_path(r"C:\Users\test\image.jpg"),
|
|
"/mnt/c/Users/test/image.jpg"
|
|
);
|
|
assert_eq!(
|
|
to_wsl_path(r"c:\projects\code\doc.txt"),
|
|
"/mnt/c/projects/code/doc.txt"
|
|
);
|
|
assert_eq!(
|
|
to_wsl_path(r"D:\Data\archive.tar"),
|
|
"/mnt/d/Data/archive.tar"
|
|
);
|
|
assert_eq!(to_wsl_path("/tmp/scratch.png"), "/tmp/scratch.png");
|
|
}
|
|
|
|
#[test]
|
|
fn test_html_to_markdown_conversion() {
|
|
let html_snippet = "Version:0.9\r\nStartHTML:00000001\r\n<!--StartFragment--><table><tr><th>Header 1</th><th>Header 2</th></tr><tr><td>Row 1 Cell 1</td><td>Row 1 Cell 2</td></tr></table><!--EndFragment-->";
|
|
let fragment = extract_html_fragment(html_snippet);
|
|
let markdown = html_to_markdown(fragment);
|
|
assert!(markdown.contains("| Header 1 | Header 2 |"));
|
|
assert!(markdown.contains("| --- | --- |"));
|
|
assert!(markdown.contains("| Row 1 Cell 1 | Row 1 Cell 2 |"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_interleaved_hotchpotch_clipboard_cache() {
|
|
let dir = tempdir().unwrap();
|
|
let state = Arc::new(MemoryState::new(dir.path().to_str().unwrap()));
|
|
let handler = ClipboardHandler;
|
|
|
|
// Step 1: Push Image A into cache
|
|
{
|
|
let mut cache = state.clipboard_cache.write().await;
|
|
cache.push_image(crate::state::CachedClipboardImage {
|
|
file_path: "C:\\tmp\\image_a.jpg".into(),
|
|
file_path_wsl: "/mnt/c/tmp/image_a.jpg".into(),
|
|
captured_at_epoch_ms: 1000,
|
|
age: "0s ago".into(),
|
|
width: 800,
|
|
height: 600,
|
|
size_bytes: 1024,
|
|
ocr_text: Some("Error: Failed to connect to server".into()),
|
|
});
|
|
}
|
|
|
|
// Step 2: Push Text 1 into cache
|
|
{
|
|
let mut cache = state.clipboard_cache.write().await;
|
|
cache.push_text(crate::state::CachedClipboardText {
|
|
text: "curl http://localhost:8080".into(),
|
|
captured_at_epoch_ms: 2000,
|
|
age: "0s ago".into(),
|
|
char_count: 26,
|
|
});
|
|
}
|
|
|
|
// Step 3: Push Image B into cache
|
|
{
|
|
let mut cache = state.clipboard_cache.write().await;
|
|
cache.push_image(crate::state::CachedClipboardImage {
|
|
file_path: "C:\\tmp\\image_b.jpg".into(),
|
|
file_path_wsl: "/mnt/c/tmp/image_b.jpg".into(),
|
|
captured_at_epoch_ms: 3000,
|
|
age: "0s ago".into(),
|
|
width: 1200,
|
|
height: 800,
|
|
size_bytes: 2048,
|
|
ocr_text: Some("git commit -m 'feat: smart clipboard'".into()),
|
|
});
|
|
}
|
|
|
|
// Step 4: Push Text 2 into cache (User got sidetracked and copied text!)
|
|
{
|
|
let mut cache = state.clipboard_cache.write().await;
|
|
cache.push_text(crate::state::CachedClipboardText {
|
|
text: "Some random stackoverflow URL".into(),
|
|
captured_at_epoch_ms: 4000,
|
|
age: "0s ago".into(),
|
|
char_count: 29,
|
|
});
|
|
}
|
|
// Now verify: Action 'image' MUST still retrieve Image B, not be shadowed by Text 2!
|
|
let img_res_str = handler
|
|
.execute(json!({ "action": "image" }), Arc::clone(&state))
|
|
.await
|
|
.unwrap();
|
|
let img_val: Value = serde_yaml::from_str(&img_res_str).unwrap();
|
|
assert_eq!(img_val["file_path"], "C:\\tmp\\image_b.jpg");
|
|
assert_eq!(img_val["file_path_wsl"], "/mnt/c/tmp/image_b.jpg");
|
|
assert_eq!(img_val["width"], 1200);
|
|
assert_eq!(img_val["height"], 800);
|
|
assert_eq!(img_val["ocr_text"], "git commit -m 'feat: smart clipboard'");
|
|
assert!(img_val.get("age").is_some());
|
|
|
|
// Action 'text' MUST retrieve Text 2
|
|
let text_res_str = handler
|
|
.execute(json!({ "action": "text" }), Arc::clone(&state))
|
|
.await
|
|
.unwrap();
|
|
let text_val: Value = serde_yaml::from_str(&text_res_str).unwrap();
|
|
assert_eq!(text_val["text"], "Some random stackoverflow URL");
|
|
|
|
// Action 'history' MUST show both items
|
|
let hist_res_str = handler
|
|
.execute(json!({ "action": "history" }), Arc::clone(&state))
|
|
.await
|
|
.unwrap();
|
|
let hist_val: Value = serde_yaml::from_str(&hist_res_str).unwrap();
|
|
let history = hist_val.as_array().unwrap();
|
|
assert!(history.len() >= 4);
|
|
}
|
|
}
|