feat: event-driven parity, Windows shell detection, dirty build timestamps, and gatekeeper integration
This commit is contained in:
Riz Ashraf committed
1 parent
64857f9d5e
commit
f4dcf3acbf
33 files changed
+3501
-143
No files matched your search
+801
-18
@@ -8,9 +8,229 @@ use serde_json::{Value, json};
|
||||
use std::borrow::Cow;
|
||||
use std::sync::Arc;
|
||||
|
||||
|
||||
static CLIPBOARD_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
pub fn to_wsl_path(win_path: &str) -> String {
|
||||
let p = win_path.trim();
|
||||
if let Some(rest) = p.strip_prefix("C:\\").or_else(|| p.strip_prefix("c:\\")) {
|
||||
format!("/mnt/c/{}", rest.replace('\\', "/"))
|
||||
} else if let Some(rest) = p.strip_prefix("D:\\").or_else(|| p.strip_prefix("d:\\")) {
|
||||
format!("/mnt/d/{}", rest.replace('\\', "/"))
|
||||
} else {
|
||||
p.replace('\\', "/")
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extract_html_fragment(raw: &str) -> &str {
|
||||
if let Some(start_idx) = raw.find("<!--StartFragment-->") {
|
||||
let after_start = start_idx + "<!--StartFragment-->".len();
|
||||
if let Some(end_idx) = raw[after_start..].find("<!--EndFragment-->") {
|
||||
return &raw[after_start..after_start + end_idx];
|
||||
}
|
||||
return &raw[after_start..];
|
||||
}
|
||||
raw
|
||||
}
|
||||
|
||||
pub fn html_to_markdown(html: &str) -> String {
|
||||
let mut out = String::new();
|
||||
let mut in_table = false;
|
||||
let mut table_rows: Vec<Vec<String>> = Vec::new();
|
||||
let mut current_row: Vec<String> = Vec::new();
|
||||
let mut current_cell = String::new();
|
||||
let mut in_cell = false;
|
||||
|
||||
let mut chars = html.chars().peekable();
|
||||
while let Some(c) = chars.next() {
|
||||
if c == '<' {
|
||||
let mut tag = String::new();
|
||||
while let Some(&next_c) = chars.peek() {
|
||||
chars.next();
|
||||
if next_c == '>' {
|
||||
break;
|
||||
}
|
||||
tag.push(next_c);
|
||||
}
|
||||
let tag_lower = tag.trim().to_lowercase();
|
||||
let tag_name = tag_lower.split_whitespace().next().unwrap_or("");
|
||||
|
||||
match tag_name {
|
||||
"table" => {
|
||||
in_table = true;
|
||||
table_rows.clear();
|
||||
}
|
||||
"/table" => {
|
||||
if in_table {
|
||||
in_table = false;
|
||||
if !table_rows.is_empty() {
|
||||
out.push('\n');
|
||||
let num_cols = table_rows.iter().map(|r| r.len()).max().unwrap_or(0);
|
||||
for (row_idx, row) in table_rows.iter().enumerate() {
|
||||
out.push('|');
|
||||
for col_idx in 0..num_cols {
|
||||
let cell_val = row.get(col_idx).map(|s| s.trim()).unwrap_or("");
|
||||
out.push(' ');
|
||||
out.push_str(cell_val);
|
||||
out.push_str(" |");
|
||||
}
|
||||
out.push('\n');
|
||||
if row_idx == 0 {
|
||||
out.push('|');
|
||||
for _ in 0..num_cols {
|
||||
out.push_str(" --- |");
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
"tr" => {
|
||||
if in_table {
|
||||
current_row.clear();
|
||||
}
|
||||
}
|
||||
"/tr" => {
|
||||
if in_table {
|
||||
if !current_row.is_empty() {
|
||||
table_rows.push(current_row.clone());
|
||||
}
|
||||
current_row.clear();
|
||||
}
|
||||
}
|
||||
"th" | "td" => {
|
||||
if in_table {
|
||||
in_cell = true;
|
||||
current_cell.clear();
|
||||
}
|
||||
}
|
||||
"/th" | "/td" => {
|
||||
if in_table && in_cell {
|
||||
in_cell = false;
|
||||
current_row.push(current_cell.trim().to_string());
|
||||
current_cell.clear();
|
||||
}
|
||||
}
|
||||
"br" | "br/" => {
|
||||
if in_cell {
|
||||
current_cell.push(' ');
|
||||
} else {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
"p" | "div" => {
|
||||
if !in_cell && !out.ends_with("\n\n") && !out.is_empty() {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
"/p" | "/div" => {
|
||||
if !in_cell && !out.ends_with("\n\n") {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
|
||||
if !in_cell {
|
||||
if !out.ends_with('\n') && !out.is_empty() {
|
||||
out.push('\n');
|
||||
}
|
||||
let level = tag_name[1..].parse::<usize>().unwrap_or(1);
|
||||
out.push_str(&"#".repeat(level));
|
||||
out.push(' ');
|
||||
}
|
||||
}
|
||||
"/h1" | "/h2" | "/h3" | "/h4" | "/h5" | "/h6" => {
|
||||
if !in_cell {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
"b" | "strong" => {
|
||||
if in_cell {
|
||||
current_cell.push_str("**");
|
||||
} else {
|
||||
out.push_str("**");
|
||||
}
|
||||
}
|
||||
"/b" | "/strong" => {
|
||||
if in_cell {
|
||||
current_cell.push_str("**");
|
||||
} else {
|
||||
out.push_str("**");
|
||||
}
|
||||
}
|
||||
"i" | "em" => {
|
||||
if in_cell {
|
||||
current_cell.push('*');
|
||||
} else {
|
||||
out.push('*');
|
||||
}
|
||||
}
|
||||
"/i" | "/em" => {
|
||||
if in_cell {
|
||||
current_cell.push('*');
|
||||
} else {
|
||||
out.push('*');
|
||||
}
|
||||
}
|
||||
"code" => {
|
||||
if in_cell {
|
||||
current_cell.push('`');
|
||||
} else {
|
||||
out.push('`');
|
||||
}
|
||||
}
|
||||
"/code" => {
|
||||
if in_cell {
|
||||
current_cell.push('`');
|
||||
} else {
|
||||
out.push('`');
|
||||
}
|
||||
}
|
||||
"li" => {
|
||||
if !in_cell {
|
||||
out.push_str("* ");
|
||||
}
|
||||
}
|
||||
"/li" => {
|
||||
if !in_cell {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
} else if c == '&' {
|
||||
let mut entity = String::new();
|
||||
while let Some(&next_c) = chars.peek() {
|
||||
chars.next();
|
||||
if next_c == ';' {
|
||||
break;
|
||||
}
|
||||
entity.push(next_c);
|
||||
}
|
||||
let decoded = match entity.as_str() {
|
||||
"nbsp" => " ",
|
||||
"amp" => "&",
|
||||
"lt" => "<",
|
||||
"gt" => ">",
|
||||
"quot" => "\"",
|
||||
"apos" | "#39" => "'",
|
||||
_ => " ",
|
||||
};
|
||||
if in_cell {
|
||||
current_cell.push_str(decoded);
|
||||
} else if !in_table {
|
||||
out.push_str(decoded);
|
||||
}
|
||||
} else if in_cell {
|
||||
current_cell.push(c);
|
||||
} else if !in_table {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
|
||||
out.trim().to_string()
|
||||
}
|
||||
|
||||
pub fn get_native_clipboard_text() -> Option<String> {
|
||||
let _guard = CLIPBOARD_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for _ in 0..3 {
|
||||
@@ -25,6 +245,83 @@ pub fn get_native_clipboard_text() -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn get_windows_html_clipboard() -> Option<String> {
|
||||
use windows_sys::Win32::System::DataExchange::{
|
||||
CloseClipboard, GetClipboardData, IsClipboardFormatAvailable, OpenClipboard,
|
||||
RegisterClipboardFormatW,
|
||||
};
|
||||
use windows_sys::Win32::System::Memory::{GlobalLock, GlobalSize, GlobalUnlock};
|
||||
let format_name: Vec<u16> = "HTML Format\0".encode_utf16().collect();
|
||||
unsafe {
|
||||
let format_id = RegisterClipboardFormatW(format_name.as_ptr());
|
||||
if format_id == 0 {
|
||||
return None;
|
||||
}
|
||||
for _ in 0..3 {
|
||||
if OpenClipboard(std::ptr::null_mut()) != 0 {
|
||||
let has_format = IsClipboardFormatAvailable(format_id) != 0;
|
||||
let mut html_opt = None;
|
||||
if has_format {
|
||||
let handle = GetClipboardData(format_id);
|
||||
if !handle.is_null() {
|
||||
let size = GlobalSize(handle as _);
|
||||
if size > 0 {
|
||||
let ptr = GlobalLock(handle as _);
|
||||
if !ptr.is_null() {
|
||||
let slice = std::slice::from_raw_parts(ptr as *const u8, size);
|
||||
if let Ok(s) = std::str::from_utf8(slice) {
|
||||
html_opt = Some(s.to_string());
|
||||
}
|
||||
GlobalUnlock(handle as _);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
CloseClipboard();
|
||||
if html_opt.is_some() {
|
||||
return html_opt;
|
||||
}
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn perform_windows_ocr(img: &image::DynamicImage) -> Option<String> {
|
||||
use windows::Graphics::Imaging::BitmapDecoder;
|
||||
use windows::Media::Ocr::OcrEngine;
|
||||
use windows::Storage::Streams::{DataWriter, InMemoryRandomAccessStream};
|
||||
|
||||
let mut png_bytes = std::io::Cursor::new(Vec::new());
|
||||
img.write_to(&mut png_bytes, image::ImageFormat::Png).ok()?;
|
||||
let bytes = png_bytes.into_inner();
|
||||
|
||||
let stream = InMemoryRandomAccessStream::new().ok()?;
|
||||
let writer = DataWriter::CreateDataWriter(&stream).ok()?;
|
||||
writer.WriteBytes(&bytes).ok()?;
|
||||
writer.StoreAsync().ok()?.join().ok()?;
|
||||
writer.FlushAsync().ok()?.join().ok()?;
|
||||
stream.Seek(0).ok()?;
|
||||
|
||||
let decoder = BitmapDecoder::CreateAsync(&stream).ok()?.join().ok()?;
|
||||
let software_bitmap = decoder.GetSoftwareBitmapAsync().ok()?.join().ok()?;
|
||||
|
||||
let engine = OcrEngine::TryCreateFromUserProfileLanguages().ok()?;
|
||||
let result = engine.RecognizeAsync(&software_bitmap).ok()?.join().ok()?;
|
||||
let text = result.Text().ok()?;
|
||||
let s = text.to_string();
|
||||
let trimmed = s.trim();
|
||||
if trimmed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(trimmed.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn get_windows_png_clipboard_image() -> Option<image::DynamicImage> {
|
||||
use windows_sys::Win32::System::DataExchange::{
|
||||
@@ -67,6 +364,7 @@ fn get_windows_png_clipboard_image() -> Option<image::DynamicImage> {
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn get_native_clipboard_image() -> Option<image::DynamicImage> {
|
||||
let _guard = CLIPBOARD_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
#[cfg(target_os = "windows")]
|
||||
@@ -89,6 +387,242 @@ pub fn get_native_clipboard_image() -> Option<image::DynamicImage> {
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn process_clipboard_snapshot(state: Arc<MemoryState>) {
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis() as u64;
|
||||
|
||||
// 1. Try to read image first
|
||||
let img_opt = tokio::task::spawn_blocking(get_native_clipboard_image)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
|
||||
if let Some(mut img) = img_opt {
|
||||
let max_dim = 1440;
|
||||
if img.width() > max_dim || img.height() > max_dim {
|
||||
img = img.resize(max_dim, max_dim, FilterType::Lanczos3);
|
||||
}
|
||||
let (w, h) = (img.width(), img.height());
|
||||
let rgb_img = img.to_rgb8();
|
||||
|
||||
let mut jpeg_bytes = std::io::Cursor::new(Vec::new());
|
||||
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, 88);
|
||||
if encoder
|
||||
.encode(&rgb_img, w, h, image::ExtendedColorType::Rgb8)
|
||||
.is_ok()
|
||||
{
|
||||
let bytes = jpeg_bytes.into_inner();
|
||||
let size_bytes = bytes.len();
|
||||
|
||||
let cache_dir = dirs::home_dir()
|
||||
.unwrap_or_default()
|
||||
.join(".gemini/mcp_memory/clipboard");
|
||||
let _ = std::fs::create_dir_all(&cache_dir);
|
||||
let file_path = cache_dir.join("clipboard_latest_image.jpg");
|
||||
let _ = std::fs::write(&file_path, &bytes);
|
||||
|
||||
let path_str = file_path.to_string_lossy().to_string();
|
||||
let wsl_path = to_wsl_path(&path_str);
|
||||
|
||||
// Deterministic OCR
|
||||
let img_clone = img.clone();
|
||||
let ocr_text_opt = tokio::task::spawn_blocking(move || {
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
perform_windows_ocr(&img_clone)
|
||||
}
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
{
|
||||
None
|
||||
}
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
|
||||
// Index OCR text into Tantivy for omni_search
|
||||
if let Some(ref text) = ocr_text_opt {
|
||||
let doc_id = format!("clipboard_{}", now_ms);
|
||||
let _ = state
|
||||
.search_index
|
||||
.read()
|
||||
.await
|
||||
.index_clipboard_image(&doc_id, &path_str, text);
|
||||
}
|
||||
|
||||
let cached = crate::state::CachedClipboardImage {
|
||||
file_path: path_str,
|
||||
file_path_wsl: wsl_path,
|
||||
captured_at_epoch_ms: now_ms,
|
||||
age: "0s ago".to_string(),
|
||||
width: w,
|
||||
height: h,
|
||||
size_bytes,
|
||||
ocr_text: ocr_text_opt,
|
||||
};
|
||||
|
||||
let mut cache = state.clipboard_cache.write().await;
|
||||
cache.last_image = Some(cached.clone());
|
||||
cache
|
||||
.history
|
||||
.push_front(crate::state::ClipboardHistoryItem::Image(cached));
|
||||
if cache.history.len() > 20 {
|
||||
cache.history.pop_back();
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Try text / HTML
|
||||
let text_opt = tokio::task::spawn_blocking(|| {
|
||||
#[cfg(target_os = "windows")]
|
||||
if let Some(raw_html) = get_windows_html_clipboard() {
|
||||
let frag = extract_html_fragment(&raw_html);
|
||||
let md = html_to_markdown(frag);
|
||||
if !md.trim().is_empty() {
|
||||
return Some(md);
|
||||
}
|
||||
}
|
||||
get_native_clipboard_text()
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
|
||||
if let Some(text) = text_opt {
|
||||
let text_trim = text.trim();
|
||||
if text_trim.is_empty() {
|
||||
return;
|
||||
}
|
||||
let char_count = text_trim.chars().count();
|
||||
let cached = crate::state::CachedClipboardText {
|
||||
text: text_trim.to_string(),
|
||||
captured_at_epoch_ms: now_ms,
|
||||
age: "0s ago".to_string(),
|
||||
char_count,
|
||||
};
|
||||
|
||||
let mut cache = state.clipboard_cache.write().await;
|
||||
if let Some(ref prev) = cache.last_text {
|
||||
if prev.text == cached.text {
|
||||
return;
|
||||
}
|
||||
}
|
||||
cache.last_text = Some(cached.clone());
|
||||
cache
|
||||
.history
|
||||
.push_front(crate::state::ClipboardHistoryItem::Text(cached));
|
||||
if cache.history.len() > 20 {
|
||||
cache.history.pop_back();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
static CLIPBOARD_NOTIFY_TX: std::sync::OnceLock<tokio::sync::mpsc::UnboundedSender<()>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn spawn_clipboard_listener(state: Arc<MemoryState>) {
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<()>();
|
||||
let _ = CLIPBOARD_NOTIFY_TX.set(tx);
|
||||
|
||||
std::thread::Builder::new()
|
||||
.name("win32-clipboard-listener".to_string())
|
||||
.spawn(move || {
|
||||
use windows_sys::Win32::Foundation::*;
|
||||
use windows_sys::Win32::System::DataExchange::*;
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::*;
|
||||
|
||||
unsafe extern "system" fn wnd_proc(
|
||||
hwnd: HWND,
|
||||
msg: u32,
|
||||
wparam: WPARAM,
|
||||
lparam: LPARAM,
|
||||
) -> LRESULT {
|
||||
if msg == WM_CLIPBOARDUPDATE {
|
||||
if let Some(tx) = CLIPBOARD_NOTIFY_TX.get() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
unsafe { DefWindowProcW(hwnd, msg, wparam, lparam) }
|
||||
}
|
||||
|
||||
unsafe {
|
||||
let class_name: Vec<u16> =
|
||||
"McpMemoryClipboardWatcherClass\0".encode_utf16().collect();
|
||||
let wnd_class = WNDCLASSEXW {
|
||||
cbSize: std::mem::size_of::<WNDCLASSEXW>() as u32,
|
||||
style: 0,
|
||||
lpfnWndProc: Some(wnd_proc),
|
||||
cbClsExtra: 0,
|
||||
cbWndExtra: 0,
|
||||
hInstance: 0 as _,
|
||||
hIcon: 0 as _,
|
||||
hCursor: 0 as _,
|
||||
hbrBackground: 0 as _,
|
||||
lpszMenuName: std::ptr::null(),
|
||||
lpszClassName: class_name.as_ptr(),
|
||||
hIconSm: 0 as _,
|
||||
};
|
||||
RegisterClassExW(&wnd_class);
|
||||
|
||||
let hwnd = CreateWindowExW(
|
||||
0,
|
||||
class_name.as_ptr(),
|
||||
std::ptr::null(),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
HWND_MESSAGE,
|
||||
0 as _,
|
||||
0 as _,
|
||||
std::ptr::null(),
|
||||
);
|
||||
if hwnd == 0 as _ {
|
||||
tracing::error!("Failed to create Win32 clipboard message window");
|
||||
return;
|
||||
}
|
||||
|
||||
if AddClipboardFormatListener(hwnd) == 0 {
|
||||
tracing::error!("Failed to AddClipboardFormatListener");
|
||||
DestroyWindow(hwnd);
|
||||
return;
|
||||
}
|
||||
tracing::info!("Win32 clipboard listener registered successfully on HWND_MESSAGE.");
|
||||
|
||||
let mut msg: MSG = std::mem::zeroed();
|
||||
while GetMessageW(&mut msg, 0 as _, 0, 0) > 0 {
|
||||
TranslateMessage(&msg);
|
||||
DispatchMessageW(&msg);
|
||||
}
|
||||
|
||||
RemoveClipboardFormatListener(hwnd);
|
||||
DestroyWindow(hwnd);
|
||||
}
|
||||
})
|
||||
.expect("Failed to spawn win32-clipboard-listener thread");
|
||||
|
||||
tokio::spawn(async move {
|
||||
while let Some(()) = rx.recv().await {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(150)).await;
|
||||
while let Ok(_) = rx.try_recv() {}
|
||||
|
||||
process_clipboard_snapshot(Arc::clone(&state)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
pub fn spawn_clipboard_listener(_state: Arc<MemoryState>) {
|
||||
tracing::info!("Clipboard listener: running in headless/Linux mode (no Win32 listener).");
|
||||
}
|
||||
|
||||
pub struct ClipboardHandler;
|
||||
|
||||
#[async_trait]
|
||||
@@ -100,13 +634,154 @@ impl McpTool for ClipboardHandler {
|
||||
fn schema(&self) -> Value {
|
||||
crate::mcp::tool_def::<ClipboardTool>(
|
||||
"clipboard",
|
||||
"Consolidated OS clipboard management (read, write)",
|
||||
"Smart OS clipboard management: Retrieve screenshots ('image') with overwrite immunity and verbatim OCR, normalized text ('text'), recent history ('history'), clear, read, or write.",
|
||||
)
|
||||
}
|
||||
|
||||
async fn execute(&self, args: Value, state: Arc<MemoryState>) -> crate::error::Result<String> {
|
||||
let req: ClipboardTool = serde_json::from_value(args).map_err(|e| e.to_string())?;
|
||||
match req.action {
|
||||
ClipboardAction::Image => {
|
||||
let cached_opt = {
|
||||
let cache = state.clipboard_cache.read().await;
|
||||
cache.last_image.clone()
|
||||
};
|
||||
|
||||
let item = if let Some(mut img) = cached_opt {
|
||||
img.age = crate::state::format_age(img.captured_at_epoch_ms);
|
||||
Some(img)
|
||||
} else {
|
||||
process_clipboard_snapshot(Arc::clone(&state)).await;
|
||||
let cache = state.clipboard_cache.read().await;
|
||||
cache.last_image.clone().map(|mut img| {
|
||||
img.age = crate::state::format_age(img.captured_at_epoch_ms);
|
||||
img
|
||||
})
|
||||
};
|
||||
|
||||
let item = if item.is_none() {
|
||||
let cache_dir = dirs::home_dir()
|
||||
.unwrap_or_default()
|
||||
.join(".gemini/mcp_memory/clipboard");
|
||||
let file_path = cache_dir.join("clipboard_latest_image.jpg");
|
||||
if file_path.exists() {
|
||||
let path_str = file_path.to_string_lossy().to_string();
|
||||
let wsl_path = to_wsl_path(&path_str);
|
||||
let meta = std::fs::metadata(&file_path).ok();
|
||||
let size_bytes = meta.as_ref().map(|m| m.len() as usize).unwrap_or(0);
|
||||
let epoch_ms = meta
|
||||
.and_then(|m| m.modified().ok())
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or_default();
|
||||
Some(crate::state::CachedClipboardImage {
|
||||
file_path: path_str,
|
||||
file_path_wsl: wsl_path,
|
||||
captured_at_epoch_ms: epoch_ms,
|
||||
age: crate::state::format_age(epoch_ms),
|
||||
width: 0,
|
||||
height: 0,
|
||||
size_bytes,
|
||||
ocr_text: None,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
item
|
||||
};
|
||||
|
||||
state.record_activity(
|
||||
"clipboard",
|
||||
"Retrieved screenshot image from clipboard cache",
|
||||
None,
|
||||
);
|
||||
|
||||
if let Some(img) = item {
|
||||
Ok(serde_json::to_string_pretty(&img)?)
|
||||
} else {
|
||||
Ok(json!({
|
||||
"status": "no_image_in_clipboard",
|
||||
"message": "No screenshot or image found in clipboard cache. Take a screenshot (Win+Shift+S on Windows) and call clipboard(action: \"image\") again."
|
||||
}).to_string())
|
||||
}
|
||||
}
|
||||
ClipboardAction::Text => {
|
||||
let cached_opt = {
|
||||
let cache = state.clipboard_cache.read().await;
|
||||
cache.last_text.clone()
|
||||
};
|
||||
|
||||
let item = if let Some(mut txt) = cached_opt {
|
||||
txt.age = crate::state::format_age(txt.captured_at_epoch_ms);
|
||||
Some(txt)
|
||||
} else {
|
||||
process_clipboard_snapshot(Arc::clone(&state)).await;
|
||||
let cache = state.clipboard_cache.read().await;
|
||||
cache.last_text.clone().map(|mut txt| {
|
||||
txt.age = crate::state::format_age(txt.captured_at_epoch_ms);
|
||||
txt
|
||||
})
|
||||
};
|
||||
|
||||
state.record_activity("clipboard", "Retrieved text from clipboard cache", None);
|
||||
if let Some(txt) = item {
|
||||
Ok(serde_json::to_string_pretty(&txt)?)
|
||||
} else {
|
||||
Ok(json!({
|
||||
"text": "",
|
||||
"status": "empty",
|
||||
"message": "Clipboard does not currently contain text."
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
}
|
||||
ClipboardAction::History => {
|
||||
let history = {
|
||||
let cache = state.clipboard_cache.read().await;
|
||||
cache
|
||||
.history
|
||||
.iter()
|
||||
.map(|item| match item {
|
||||
crate::state::ClipboardHistoryItem::Image(img) => {
|
||||
let mut img = img.clone();
|
||||
img.age = crate::state::format_age(img.captured_at_epoch_ms);
|
||||
crate::state::ClipboardHistoryItem::Image(img)
|
||||
}
|
||||
crate::state::ClipboardHistoryItem::Text(txt) => {
|
||||
let mut txt = txt.clone();
|
||||
txt.age = crate::state::format_age(txt.captured_at_epoch_ms);
|
||||
crate::state::ClipboardHistoryItem::Text(txt)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
state.record_activity("clipboard", "Viewed clipboard history ring buffer", None);
|
||||
Ok(serde_json::to_string_pretty(&history)?)
|
||||
}
|
||||
ClipboardAction::Clear => {
|
||||
tokio::task::spawn_blocking(|| {
|
||||
let _guard = CLIPBOARD_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Ok(mut c) = Clipboard::new() {
|
||||
let _ = c.clear();
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|e| crate::error::AppError::Internal(e.to_string()))?;
|
||||
|
||||
let mut cache = state.clipboard_cache.write().await;
|
||||
cache.last_image = None;
|
||||
cache.last_text = None;
|
||||
cache.history.clear();
|
||||
|
||||
state.record_activity("clipboard", "Cleared OS clipboard and memory cache", None);
|
||||
Ok(json!({
|
||||
"status": "cleared",
|
||||
"message": "OS clipboard and memory cache successfully cleared."
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
ClipboardAction::Read => {
|
||||
let (mut out, b64_opt) =
|
||||
tokio::task::spawn_blocking(move || -> crate::error::Result<(serde_json::Map<String, Value>, Option<String>)> {
|
||||
@@ -119,14 +794,15 @@ impl McpTool for ClipboardHandler {
|
||||
|
||||
if let Some(dynamic_img) = get_native_clipboard_image() {
|
||||
let mut img = dynamic_img;
|
||||
let max_dim = 1024;
|
||||
let max_dim = 1440;
|
||||
if img.width() > max_dim || img.height() > max_dim {
|
||||
img = img.resize(max_dim, max_dim, FilterType::Lanczos3);
|
||||
}
|
||||
let rgb_img = img.into_rgb8();
|
||||
|
||||
let mut jpeg_bytes = std::io::Cursor::new(Vec::new());
|
||||
if rgb_img.write_to(&mut jpeg_bytes, image::ImageFormat::Jpeg).is_ok() {
|
||||
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, 88);
|
||||
if encoder.encode(&rgb_img, rgb_img.width(), rgb_img.height(), image::ExtendedColorType::Rgb8).is_ok() {
|
||||
let bytes = jpeg_bytes.into_inner();
|
||||
use base64::Engine;
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
||||
@@ -136,18 +812,12 @@ impl McpTool for ClipboardHandler {
|
||||
.unwrap_or_default()
|
||||
.join(".gemini/mcp_memory/clipboard");
|
||||
let _ = std::fs::create_dir_all(&cache_dir);
|
||||
let file_path = cache_dir.join("clipboard_latest.jpg");
|
||||
let file_path = cache_dir.join("clipboard_latest_image.jpg");
|
||||
if std::fs::write(&file_path, &bytes).is_ok() {
|
||||
let path_str = file_path.to_string_lossy().to_string();
|
||||
out.insert("image_path".into(), json!(path_str));
|
||||
#[cfg(target_os = "windows")]
|
||||
if let Some(rest) = path_str.strip_prefix("C:\\") {
|
||||
let wsl_path = format!("/mnt/c/{}", rest.replace('\\', "/"));
|
||||
out.insert("image_path_wsl".into(), json!(wsl_path));
|
||||
} else if let Some(rest) = path_str.strip_prefix("c:\\") {
|
||||
let wsl_path = format!("/mnt/c/{}", rest.replace('\\', "/"));
|
||||
out.insert("image_path_wsl".into(), json!(wsl_path));
|
||||
}
|
||||
let wsl_path = to_wsl_path(&path_str);
|
||||
out.insert("image_path_wsl".into(), json!(wsl_path));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -171,7 +841,9 @@ impl McpTool for ClipboardHandler {
|
||||
}
|
||||
|
||||
state.record_activity("clipboard", "Read contents from OS clipboard", None);
|
||||
Ok::<String, crate::error::AppError>(serde_json::to_string_pretty(&Value::Object(out))?)
|
||||
Ok::<String, crate::error::AppError>(serde_json::to_string_pretty(&Value::Object(
|
||||
out,
|
||||
))?)
|
||||
}
|
||||
ClipboardAction::Write => {
|
||||
let text_opt = req.text;
|
||||
@@ -258,7 +930,6 @@ impl McpTool for ClipboardHandler {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -266,7 +937,6 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_write_clipboard() {
|
||||
let dir = tempdir().unwrap();
|
||||
@@ -351,6 +1021,119 @@ mod tests {
|
||||
"Regression detected: vision.rs must not invoke xclip!"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wsl_path_translation() {
|
||||
assert_eq!(
|
||||
to_wsl_path(r"C:\Users\test\image.jpg"),
|
||||
"/mnt/c/Users/test/image.jpg"
|
||||
);
|
||||
assert_eq!(
|
||||
to_wsl_path(r"c:\projects\code\doc.txt"),
|
||||
"/mnt/c/projects/code/doc.txt"
|
||||
);
|
||||
assert_eq!(
|
||||
to_wsl_path(r"D:\Data\archive.tar"),
|
||||
"/mnt/d/Data/archive.tar"
|
||||
);
|
||||
assert_eq!(to_wsl_path("/tmp/scratch.png"), "/tmp/scratch.png");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_html_to_markdown_conversion() {
|
||||
let html_snippet = "Version:0.9\r\nStartHTML:00000001\r\n<!--StartFragment--><table><tr><th>Header 1</th><th>Header 2</th></tr><tr><td>Row 1 Cell 1</td><td>Row 1 Cell 2</td></tr></table><!--EndFragment-->";
|
||||
let fragment = extract_html_fragment(html_snippet);
|
||||
let markdown = html_to_markdown(fragment);
|
||||
assert!(markdown.contains("| Header 1 | Header 2 |"));
|
||||
assert!(markdown.contains("| --- | --- |"));
|
||||
assert!(markdown.contains("| Row 1 Cell 1 | Row 1 Cell 2 |"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_interleaved_hotchpotch_clipboard_cache() {
|
||||
let dir = tempdir().unwrap();
|
||||
let state = Arc::new(MemoryState::new(dir.path().to_str().unwrap()));
|
||||
let handler = ClipboardHandler;
|
||||
|
||||
// Step 1: Push Image A into cache
|
||||
{
|
||||
let mut cache = state.clipboard_cache.write().await;
|
||||
cache.push_image(crate::state::CachedClipboardImage {
|
||||
file_path: "C:\\tmp\\image_a.jpg".into(),
|
||||
file_path_wsl: "/mnt/c/tmp/image_a.jpg".into(),
|
||||
captured_at_epoch_ms: 1000,
|
||||
age: "0s ago".into(),
|
||||
width: 800,
|
||||
height: 600,
|
||||
size_bytes: 1024,
|
||||
ocr_text: Some("Error: Failed to connect to server".into()),
|
||||
});
|
||||
}
|
||||
|
||||
// Step 2: Push Text 1 into cache
|
||||
{
|
||||
let mut cache = state.clipboard_cache.write().await;
|
||||
cache.push_text(crate::state::CachedClipboardText {
|
||||
text: "curl http://localhost:8080".into(),
|
||||
captured_at_epoch_ms: 2000,
|
||||
age: "0s ago".into(),
|
||||
char_count: 26,
|
||||
});
|
||||
}
|
||||
|
||||
// Step 3: Push Image B into cache
|
||||
{
|
||||
let mut cache = state.clipboard_cache.write().await;
|
||||
cache.push_image(crate::state::CachedClipboardImage {
|
||||
file_path: "C:\\tmp\\image_b.jpg".into(),
|
||||
file_path_wsl: "/mnt/c/tmp/image_b.jpg".into(),
|
||||
captured_at_epoch_ms: 3000,
|
||||
age: "0s ago".into(),
|
||||
width: 1200,
|
||||
height: 800,
|
||||
size_bytes: 2048,
|
||||
ocr_text: Some("git commit -m 'feat: smart clipboard'".into()),
|
||||
});
|
||||
}
|
||||
|
||||
// Step 4: Push Text 2 into cache (User got sidetracked and copied text!)
|
||||
{
|
||||
let mut cache = state.clipboard_cache.write().await;
|
||||
cache.push_text(crate::state::CachedClipboardText {
|
||||
text: "Some random stackoverflow URL".into(),
|
||||
captured_at_epoch_ms: 4000,
|
||||
age: "0s ago".into(),
|
||||
char_count: 29,
|
||||
});
|
||||
}
|
||||
// Now verify: Action 'image' MUST still retrieve Image B, not be shadowed by Text 2!
|
||||
let img_res_str = handler
|
||||
.execute(json!({ "action": "image" }), Arc::clone(&state))
|
||||
.await
|
||||
.unwrap();
|
||||
let img_val: Value = serde_json::from_str(&img_res_str).unwrap();
|
||||
assert_eq!(img_val["file_path"], "C:\\tmp\\image_b.jpg");
|
||||
assert_eq!(img_val["file_path_wsl"], "/mnt/c/tmp/image_b.jpg");
|
||||
assert_eq!(img_val["width"], 1200);
|
||||
assert_eq!(img_val["height"], 800);
|
||||
assert_eq!(img_val["ocr_text"], "git commit -m 'feat: smart clipboard'");
|
||||
assert!(img_val.get("age").is_some());
|
||||
|
||||
// Action 'text' MUST retrieve Text 2
|
||||
let text_res_str = handler
|
||||
.execute(json!({ "action": "text" }), Arc::clone(&state))
|
||||
.await
|
||||
.unwrap();
|
||||
let text_val: Value = serde_json::from_str(&text_res_str).unwrap();
|
||||
assert_eq!(text_val["text"], "Some random stackoverflow URL");
|
||||
|
||||
// Action 'history' MUST show both items
|
||||
let hist_res_str = handler
|
||||
.execute(json!({ "action": "history" }), Arc::clone(&state))
|
||||
.await
|
||||
.unwrap();
|
||||
let hist_val: Value = serde_json::from_str(&hist_res_str).unwrap();
|
||||
let history = hist_val.as_array().unwrap();
|
||||
assert!(history.len() >= 4);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user