feat: implement hybrid fetch_url tool and isolate crashes
This commit is contained in:
Riz Ashraf committed
1 parent
4342615649
commit
3533c243d2
9 files changed
+447
-60
No files matched your search
Generated
+228
-12
@@ -8,6 +8,23 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "agentic-tools-rs"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"htmd",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"tiktoken-rs",
|
||||
"tokio",
|
||||
"tree-sitter",
|
||||
"tree-sitter-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
@@ -581,7 +598,7 @@ dependencies = [
|
||||
"byteorder",
|
||||
"candle-core",
|
||||
"candle-nn",
|
||||
"fancy-regex",
|
||||
"fancy-regex 0.18.0",
|
||||
"num-traits",
|
||||
"rand 0.9.5",
|
||||
"rayon",
|
||||
@@ -1327,6 +1344,17 @@ dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.19.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastdivide"
|
||||
version = "0.4.2"
|
||||
@@ -1910,6 +1938,27 @@ dependencies = [
|
||||
"xet-runtime",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "htmd"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5a1c7113c831fec68cbd79cd8bf281a84e5b6943f51473dc266b0b88a6a017e"
|
||||
dependencies = [
|
||||
"html5ever",
|
||||
"markup5ever_rcdom",
|
||||
"phf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "htmlescape"
|
||||
version = "0.3.1"
|
||||
@@ -2600,6 +2649,29 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c"
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril",
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever_rcdom"
|
||||
version = "0.38.0+unofficial"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "333171ccdf66e915257740d44e38ea5b1b19ce7b45d33cc35cb6f118fbd981ff"
|
||||
dependencies = [
|
||||
"html5ever",
|
||||
"markup5ever",
|
||||
"tendril",
|
||||
"xml5ever",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.2.0"
|
||||
@@ -2629,6 +2701,7 @@ dependencies = [
|
||||
name = "mcp-memory-server"
|
||||
version = "1.0.0"
|
||||
dependencies = [
|
||||
"agentic-tools-rs",
|
||||
"arboard",
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -3292,6 +3365,59 @@ version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project"
|
||||
version = "1.1.13"
|
||||
@@ -3361,6 +3487,12 @@ dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "precomputed-hash"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.3.0"
|
||||
@@ -4325,10 +4457,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
version = "1.0.152"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
checksum = "1741ab7a6cc54a03a89b5d563ed60075c277d9e3cfa73ad0c1f23f23974703c6"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
@@ -4498,6 +4631,12 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
|
||||
|
||||
[[package]]
|
||||
name = "sketches-ddsketch"
|
||||
version = "0.4.1"
|
||||
@@ -4563,6 +4702,37 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"parking_lot",
|
||||
"phf_shared",
|
||||
"precomputed-hash",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "string_cache_codegen"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
@@ -4834,6 +5004,15 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.69"
|
||||
@@ -4897,6 +5076,20 @@ dependencies = [
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiktoken-rs"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2aeff724640cfe13037336ddf35befdffd2909cbdb65cf041cc8a4cf8c584cfa"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
"bstr",
|
||||
"fancy-regex 0.19.2",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.55"
|
||||
@@ -5011,9 +5204,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.53.1"
|
||||
version = "1.53.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
|
||||
checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
@@ -5277,13 +5470,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter"
|
||||
version = "0.23.2"
|
||||
version = "0.27.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0203df02a3b6dd63575cc1d6e609edc2181c9a11867a271b25cfd2abff3ec5ca"
|
||||
checksum = "a5129de2ccc97808c4583d48f1fb44576d015cf241e432800572bda11ef30c3b"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"serde_json",
|
||||
"streaming-iterator",
|
||||
"tree-sitter-language",
|
||||
]
|
||||
|
||||
@@ -5329,9 +5523,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-language"
|
||||
version = "0.1.8"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081"
|
||||
checksum = "d0af592be68c579aa78a16846bd19422978c3c52e438523d45ff5d1bff1f9d4a"
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-python"
|
||||
@@ -5345,9 +5539,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tree-sitter-rust"
|
||||
version = "0.23.3"
|
||||
version = "0.24.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca8ccb3e3a3495c8a943f6c3fd24c3804c471fd7f4f16087623c7fa4c0068e8a"
|
||||
checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"tree-sitter-language",
|
||||
@@ -5718,6 +5912,18 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web_atoms"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3"
|
||||
dependencies = [
|
||||
"phf",
|
||||
"phf_codegen",
|
||||
"string_cache",
|
||||
"string_cache_codegen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.9"
|
||||
@@ -6170,6 +6376,16 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xml5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3dc9559429edf0cd3f327cc0afd9d6b36fa8cec6d93107b7fbe64f806b5f2d9"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "y4m"
|
||||
version = "0.8.0"
|
||||
|
||||
+3
-2
@@ -36,8 +36,8 @@ arboard = "3.6.1"
|
||||
image = "0.25.10"
|
||||
base64 = "0.23.1"
|
||||
git2 = "0.19.0"
|
||||
tree-sitter = "0.23.2"
|
||||
tree-sitter-rust = "0.23.3"
|
||||
tree-sitter = "0.27.1"
|
||||
tree-sitter-rust = "0.24.2"
|
||||
tree-sitter-typescript = "0.23.2"
|
||||
tree-sitter-python = "0.23.6"
|
||||
tree-sitter-java = "0.23.5"
|
||||
@@ -52,6 +52,7 @@ chrono = { version = "0.4.45", features = ["serde"] }
|
||||
ocrs = "0.13.1"
|
||||
rten = "0.26.0"
|
||||
serde_yaml = "0.9.34"
|
||||
agentic-tools-rs = { version = "0.1.0", path = "../../agentic-tools-rs" }
|
||||
|
||||
[build-dependencies]
|
||||
chrono = "0.4.45"
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
use mcp_memory_server::embedding::generate_embeddings_async;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("Testing candle embeddings...");
|
||||
let res = generate_embeddings_async(vec!["hello world".to_string()]).await;
|
||||
match res {
|
||||
Ok(v) => println!("Success: vector length {}", v.len()),
|
||||
Err(e) => println!("Error: {}", e),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn main() {
|
||||
println!("Starting Win32 clipboard listener isolation test");
|
||||
|
||||
let (_tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<()>();
|
||||
|
||||
std::thread::Builder::new()
|
||||
.name("win32-clipboard-listener-isolation".to_string())
|
||||
.spawn(move || {
|
||||
use windows_sys::Win32::Foundation::*;
|
||||
use windows_sys::Win32::System::DataExchange::*;
|
||||
use windows_sys::Win32::UI::WindowsAndMessaging::*;
|
||||
|
||||
unsafe extern "system" fn wnd_proc(
|
||||
hwnd: HWND,
|
||||
msg: u32,
|
||||
wparam: WPARAM,
|
||||
lparam: LPARAM,
|
||||
) -> LRESULT {
|
||||
if msg == WM_CLIPBOARDUPDATE {
|
||||
println!("WM_CLIPBOARDUPDATE received");
|
||||
return 0;
|
||||
}
|
||||
unsafe { DefWindowProcW(hwnd, msg, wparam, lparam) }
|
||||
}
|
||||
|
||||
unsafe {
|
||||
let class_name: Vec<u16> =
|
||||
"McpMemoryClipboardWatcherClassIsolation\0".encode_utf16().collect();
|
||||
let wnd_class = WNDCLASSEXW {
|
||||
cbSize: std::mem::size_of::<WNDCLASSEXW>() as u32,
|
||||
style: 0,
|
||||
lpfnWndProc: Some(wnd_proc),
|
||||
cbClsExtra: 0,
|
||||
cbWndExtra: 0,
|
||||
hInstance: 0 as _,
|
||||
hIcon: 0 as _,
|
||||
hCursor: 0 as _,
|
||||
hbrBackground: 0 as _,
|
||||
lpszMenuName: std::ptr::null(),
|
||||
lpszClassName: class_name.as_ptr(),
|
||||
hIconSm: 0 as _,
|
||||
};
|
||||
let class_atom = RegisterClassExW(&wnd_class);
|
||||
if class_atom == 0 {
|
||||
println!("Failed to RegisterClassExW");
|
||||
}
|
||||
|
||||
let hwnd = CreateWindowExW(
|
||||
0,
|
||||
class_name.as_ptr(),
|
||||
std::ptr::null(),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
HWND_MESSAGE,
|
||||
0 as _,
|
||||
0 as _,
|
||||
std::ptr::null(),
|
||||
);
|
||||
if hwnd == 0 as _ {
|
||||
println!("Failed to create Win32 clipboard message window");
|
||||
return;
|
||||
}
|
||||
|
||||
if AddClipboardFormatListener(hwnd) == 0 {
|
||||
println!("Failed to AddClipboardFormatListener");
|
||||
DestroyWindow(hwnd);
|
||||
return;
|
||||
}
|
||||
println!("Win32 clipboard listener registered successfully on HWND_MESSAGE.");
|
||||
|
||||
let mut msg: MSG = std::mem::zeroed();
|
||||
while GetMessageW(&mut msg, 0 as _, 0, 0) > 0 {
|
||||
TranslateMessage(&msg);
|
||||
DispatchMessageW(&msg);
|
||||
}
|
||||
|
||||
RemoveClipboardFormatListener(hwnd);
|
||||
DestroyWindow(hwnd);
|
||||
}
|
||||
})
|
||||
.expect("Failed to spawn thread");
|
||||
|
||||
// Run a basic tokio runtime to keep the main thread alive for a bit
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
rt.block_on(async {
|
||||
let _ = tokio::time::timeout(tokio::time::Duration::from_secs(3), async {
|
||||
while let Some(()) = rx.recv().await {
|
||||
println!("Got message from channel");
|
||||
}
|
||||
}).await;
|
||||
});
|
||||
|
||||
println!("Isolation test completed without crash.");
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
fn main() {
|
||||
println!("Not on windows, skipping isolation test");
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
fn main() {
|
||||
println!("Testing tree-sitter language init...");
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
|
||||
println!("Testing Rust...");
|
||||
parser.set_language(&tree_sitter_rust::LANGUAGE.into()).unwrap();
|
||||
parser.parse("fn main() {}", None).unwrap();
|
||||
|
||||
println!("Testing TypeScript...");
|
||||
parser.set_language(&tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()).unwrap();
|
||||
parser.parse("const x: number = 1;", None).unwrap();
|
||||
|
||||
println!("Testing Python...");
|
||||
parser.set_language(&tree_sitter_python::LANGUAGE.into()).unwrap();
|
||||
parser.parse("def main(): pass", None).unwrap();
|
||||
|
||||
println!("Testing Java...");
|
||||
parser.set_language(&tree_sitter_java::LANGUAGE.into()).unwrap();
|
||||
parser.parse("class Main {}", None).unwrap();
|
||||
|
||||
println!("Testing C...");
|
||||
parser.set_language(&tree_sitter_c::LANGUAGE.into()).unwrap();
|
||||
parser.parse("int main() {}", None).unwrap();
|
||||
|
||||
println!("Testing C++...");
|
||||
parser.set_language(&tree_sitter_cpp::LANGUAGE.into()).unwrap();
|
||||
parser.parse("int main() {}", None).unwrap();
|
||||
|
||||
println!("Testing Go...");
|
||||
parser.set_language(&tree_sitter_go::LANGUAGE.into()).unwrap();
|
||||
parser.parse("func main() {}", None).unwrap();
|
||||
|
||||
println!("All languages parsed successfully.");
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
use mcp_memory_server::state::MemoryState;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("Testing indexer on current workspace...");
|
||||
let state = Arc::new(MemoryState::new_in_memory());
|
||||
mcp_memory_server::indexer::start_background_indexer(state).await;
|
||||
// sleep a bit to let it run
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(10)).await;
|
||||
println!("Done!");
|
||||
}
|
||||
+45
-45
@@ -2,7 +2,6 @@ use crate::router::McpTool;
|
||||
use crate::state::MemoryState;
|
||||
use crate::tools::FetchUrlTool;
|
||||
use async_trait::async_trait;
|
||||
use reqwest::Client;
|
||||
use serde_json::Value;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -17,7 +16,7 @@ impl McpTool for FetchUrlHandler {
|
||||
fn schema(&self) -> Value {
|
||||
crate::mcp::tool_def::<FetchUrlTool>(
|
||||
"fetch_url",
|
||||
"Fetch content from a URL via an HTTP GET request natively (supports proxy config via environment variables: HTTP_PROXY, HTTPS_PROXY, NO_PROXY)",
|
||||
"Fetch content from a URL via an HTTP request natively (supports proxy config via environment variables: HTTP_PROXY, HTTPS_PROXY, NO_PROXY)",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -29,54 +28,31 @@ impl McpTool for FetchUrlHandler {
|
||||
))
|
||||
})?;
|
||||
|
||||
let mut client_builder = Client::builder();
|
||||
let method_str = req.method.unwrap_or_else(|| "GET".to_string());
|
||||
|
||||
// Add optional user agent
|
||||
if let Some(ua) = req.user_agent {
|
||||
client_builder = client_builder.user_agent(ua);
|
||||
} else {
|
||||
client_builder = client_builder.user_agent("mcp-memory-server/1.0");
|
||||
}
|
||||
let headers: Vec<String> = req.headers.unwrap_or_default().into_iter()
|
||||
.map(|(k, v)| format!("{}={}", k, v))
|
||||
.collect();
|
||||
|
||||
// Allow bypassing SSL validation for self-signed certificates
|
||||
if req.ignore_ssl_errors.unwrap_or(false) {
|
||||
client_builder = client_builder.danger_accept_invalid_certs(true);
|
||||
}
|
||||
|
||||
// Reqwest automatically uses HTTP_PROXY, HTTPS_PROXY, NO_PROXY
|
||||
// environment variables by default, so we don't need to manually
|
||||
// extract and apply them, the builder handles it.
|
||||
|
||||
let client = client_builder.build().map_err(|e| {
|
||||
crate::error::AppError::Internal(format!("Failed to build HTTP client: {}", e))
|
||||
// Pass to our agentic-tools-rs library directly
|
||||
let res = agentic_tools_rs::commands::llmfetch::run(
|
||||
&req.url,
|
||||
&method_str,
|
||||
req.body.as_ref(),
|
||||
&headers,
|
||||
true, // Always condense for MCP
|
||||
).await.map_err(|e| {
|
||||
crate::error::AppError::Internal(format!("Agentic fetch failed: {}", e))
|
||||
})?;
|
||||
|
||||
let res = client.get(&req.url).send().await.map_err(|e| {
|
||||
crate::error::AppError::Internal(format!("HTTP request to {} failed: {}", req.url, e))
|
||||
})?;
|
||||
|
||||
let status = res.status();
|
||||
let content = res.text().await.map_err(|e| {
|
||||
crate::error::AppError::Internal(format!("Failed to read response body: {}", e))
|
||||
})?;
|
||||
|
||||
if !status.is_success() {
|
||||
return Ok(format!(
|
||||
"HTTP Error: {} {}\n\nResponse Body:\n{}",
|
||||
status.as_u16(),
|
||||
status.canonical_reason().unwrap_or("Unknown"),
|
||||
content
|
||||
));
|
||||
}
|
||||
|
||||
Ok(content)
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{routing::get, Router};
|
||||
use axum::{routing::{get, post}, Router};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
async fn spawn_test_server() -> String {
|
||||
@@ -90,7 +66,8 @@ mod tests {
|
||||
"Not Found Error",
|
||||
)
|
||||
}),
|
||||
);
|
||||
)
|
||||
.route("/echo", post(|body: String| async move { format!("ECHO: {}", body) }));
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
@@ -113,12 +90,13 @@ mod tests {
|
||||
serde_json::json!({
|
||||
"url": url
|
||||
}),
|
||||
Arc::new(MemoryState::new("")),
|
||||
Arc::new(MemoryState::new_in_memory()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(res, "Hello, World!");
|
||||
assert!(res.contains("HTTP Status: 200 OK"));
|
||||
assert!(res.contains("Hello, World!"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -132,12 +110,34 @@ mod tests {
|
||||
serde_json::json!({
|
||||
"url": url
|
||||
}),
|
||||
Arc::new(MemoryState::new("")),
|
||||
Arc::new(MemoryState::new_in_memory()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(res.contains("HTTP Error: 404 Not Found"));
|
||||
assert!(res.contains("HTTP Status: 404 Not Found"));
|
||||
assert!(res.contains("Not Found Error"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fetch_url_post() {
|
||||
let base_url = spawn_test_server().await;
|
||||
let handler = FetchUrlHandler;
|
||||
let url = format!("{}/echo", base_url);
|
||||
|
||||
let res = handler
|
||||
.execute(
|
||||
serde_json::json!({
|
||||
"url": url,
|
||||
"method": "POST",
|
||||
"body": "test_body_content"
|
||||
}),
|
||||
Arc::new(MemoryState::new_in_memory()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(res.contains("HTTP Status: 200 OK"));
|
||||
assert!(res.contains("ECHO: test_body_content"));
|
||||
}
|
||||
}
|
||||
+7
-1
@@ -1040,7 +1040,7 @@ pub struct ClipboardTool {
|
||||
pub image_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Fetch content from a URL via an HTTP GET request natively (supports proxy config via environment variables: HTTP_PROXY, HTTPS_PROXY, NO_PROXY).
|
||||
/// Fetch content from a URL via an HTTP request natively (supports proxy config via environment variables: HTTP_PROXY, HTTPS_PROXY, NO_PROXY).
|
||||
#[derive(Debug, Deserialize, Serialize, JsonSchema)]
|
||||
pub struct FetchUrlTool {
|
||||
/// URL to fetch content from.
|
||||
@@ -1049,4 +1049,10 @@ pub struct FetchUrlTool {
|
||||
pub user_agent: Option<String>,
|
||||
/// Bypass SSL certificate verification for self-signed or invalid certs. Use with caution.
|
||||
pub ignore_ssl_errors: Option<bool>,
|
||||
/// HTTP method to use (e.g., "GET", "POST"). Defaults to "GET".
|
||||
pub method: Option<String>,
|
||||
/// Optional headers for the request.
|
||||
pub headers: Option<std::collections::HashMap<String, String>>,
|
||||
/// Optional body for POST/PUT requests, typically a string or JSON string.
|
||||
pub body: Option<String>,
|
||||
}
|
||||
@@ -236,7 +236,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn test_spawn_watcher_invalid_path() {
|
||||
unsafe { std::env::remove_var("MCP_ALLOW_TMP_FALLBACK") };
|
||||
let result = std::panic::catch_unwind(|| {
|
||||
let state = Arc::new(MemoryState::new("/nonexistent/path"));
|
||||
spawn_watcher(state);
|
||||
|
||||
Reference in new issue
Block a user