feat: add FetchUrlTool with proxy support and cross-OS path sanitization
- Implement native FetchUrlTool using reqwest for HTTP fetching, which respects HTTP_PROXY, HTTPS_PROXY, and NO_PROXY natively. - Introduce sanitize_path in utils to bidirectionally handle Windows/WSL path mapping, applying it to various search/git handlers. - Update .gitignore to exclude mcp_store.redb and tantivy_index.
This commit is contained in:
1 parent
250b70d329
commit
5583132cdc
10 files changed
+191
-5
No files matched your search
@@ -10,3 +10,5 @@ temp.json
|
|||||||
*.txt
|
*.txt
|
||||||
summary.md
|
summary.md
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
server/mcp_store.redb
|
||||||
|
server/tantivy_index/
|
||||||
+1
-1
@@ -14,7 +14,7 @@ clap = { version = "4.6.6", features = ["derive"] }
|
|||||||
dirs = "7.0.0"
|
dirs = "7.0.0"
|
||||||
futures-util = "0.3.34"
|
futures-util = "0.3.34"
|
||||||
redb = "4.3.0"
|
redb = "4.3.0"
|
||||||
reqwest = { version = "0.13", default-features = false, features = ["stream", "json"] }
|
reqwest = { version = "0.13", default-features = false, features = ["stream", "json", "rustls"] }
|
||||||
schemars = "1.2.2"
|
schemars = "1.2.2"
|
||||||
serde = { version = "1.0.229", features = ["derive"] }
|
serde = { version = "1.0.229", features = ["derive"] }
|
||||||
serde_json = "1.0.151"
|
serde_json = "1.0.151"
|
||||||
|
|||||||
@@ -440,7 +440,7 @@ fn scan_workspace_for_symbol(
|
|||||||
|
|
||||||
if is_match {
|
if is_match {
|
||||||
results.push(serde_json::json!({
|
results.push(serde_json::json!({
|
||||||
"file_path": path.to_string_lossy(),
|
"file_path": crate::handlers::utils::sanitize_path(&path.to_string_lossy()),
|
||||||
"line": line_num + 1,
|
"line": line_num + 1,
|
||||||
"content": line.trim(),
|
"content": line.trim(),
|
||||||
}));
|
}));
|
||||||
|
|||||||
@@ -66,7 +66,8 @@ impl McpTool for GetActiveWorktreeContextHandler {
|
|||||||
|
|
||||||
for delta in diff.deltas() {
|
for delta in diff.deltas() {
|
||||||
if let Some(path) = delta.new_file().path() {
|
if let Some(path) = delta.new_file().path() {
|
||||||
files.push(path.to_string_lossy().into_owned());
|
let path_str = path.to_string_lossy().into_owned();
|
||||||
|
files.push(crate::handlers::utils::sanitize_path(&path_str));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -121,7 +122,7 @@ impl McpTool for QueryGitDiffsHandler {
|
|||||||
|| change.git_commit.as_ref().is_some_and(|c| c.contains(&q))
|
|| change.git_commit.as_ref().is_some_and(|c| c.contains(&q))
|
||||||
{
|
{
|
||||||
matches.push(json!({
|
matches.push(json!({
|
||||||
"file_path": change.file_path,
|
"file_path": crate::handlers::utils::sanitize_path(&change.file_path),
|
||||||
"description": change.description,
|
"description": change.description,
|
||||||
"commit": change.git_commit,
|
"commit": change.git_commit,
|
||||||
"branch": change.git_branch,
|
"branch": change.git_branch,
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
use crate::router::McpTool;
|
||||||
|
use crate::state::MemoryState;
|
||||||
|
use crate::tools::FetchUrlTool;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use reqwest::Client;
|
||||||
|
use serde_json::Value;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
pub struct FetchUrlHandler;
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl McpTool for FetchUrlHandler {
|
||||||
|
fn name(&self) -> &'static str {
|
||||||
|
"fetch_url"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn schema(&self) -> Value {
|
||||||
|
crate::mcp::tool_def::<FetchUrlTool>(
|
||||||
|
"fetch_url",
|
||||||
|
"Fetch content from a URL via an HTTP GET request natively (supports proxy config via environment variables: HTTP_PROXY, HTTPS_PROXY, NO_PROXY)",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn execute(&self, args: Value, _state: Arc<MemoryState>) -> crate::error::Result<String> {
|
||||||
|
let req: FetchUrlTool = serde_json::from_value(args).map_err(|e| {
|
||||||
|
crate::error::AppError::BadRequest(format!(
|
||||||
|
"Schema validation failed. Your JSON arguments do not match the expected tool schema: {}",
|
||||||
|
e
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut client_builder = Client::builder();
|
||||||
|
|
||||||
|
// Add optional user agent
|
||||||
|
if let Some(ua) = req.user_agent {
|
||||||
|
client_builder = client_builder.user_agent(ua);
|
||||||
|
} else {
|
||||||
|
client_builder = client_builder.user_agent("mcp-memory-server/1.0");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reqwest automatically uses HTTP_PROXY, HTTPS_PROXY, NO_PROXY
|
||||||
|
// environment variables by default, so we don't need to manually
|
||||||
|
// extract and apply them, the builder handles it.
|
||||||
|
|
||||||
|
let client = client_builder.build().map_err(|e| {
|
||||||
|
crate::error::AppError::Internal(format!("Failed to build HTTP client: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let res = client.get(&req.url).send().await.map_err(|e| {
|
||||||
|
crate::error::AppError::Internal(format!("HTTP request to {} failed: {}", req.url, e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let status = res.status();
|
||||||
|
let content = res.text().await.map_err(|e| {
|
||||||
|
crate::error::AppError::Internal(format!("Failed to read response body: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if !status.is_success() {
|
||||||
|
return Ok(format!(
|
||||||
|
"HTTP Error: {} {}\n\nResponse Body:\n{}",
|
||||||
|
status.as_u16(),
|
||||||
|
status.canonical_reason().unwrap_or("Unknown"),
|
||||||
|
content
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use axum::{routing::get, Router};
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
async fn spawn_test_server() -> String {
|
||||||
|
let app = Router::new()
|
||||||
|
.route("/test", get(|| async { "Hello, World!" }))
|
||||||
|
.route(
|
||||||
|
"/not_found",
|
||||||
|
get(|| async {
|
||||||
|
(
|
||||||
|
axum::http::StatusCode::NOT_FOUND,
|
||||||
|
"Not Found Error",
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let port = listener.local_addr().unwrap().port();
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
});
|
||||||
|
|
||||||
|
format!("http://127.0.0.1:{}", port)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_fetch_url_success() {
|
||||||
|
let base_url = spawn_test_server().await;
|
||||||
|
let handler = FetchUrlHandler;
|
||||||
|
let url = format!("{}/test", base_url);
|
||||||
|
|
||||||
|
let res = handler
|
||||||
|
.execute(
|
||||||
|
serde_json::json!({
|
||||||
|
"url": url
|
||||||
|
}),
|
||||||
|
Arc::new(MemoryState::new("")),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(res, "Hello, World!");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_fetch_url_not_found() {
|
||||||
|
let base_url = spawn_test_server().await;
|
||||||
|
let handler = FetchUrlHandler;
|
||||||
|
let url = format!("{}/not_found", base_url);
|
||||||
|
|
||||||
|
let res = handler
|
||||||
|
.execute(
|
||||||
|
serde_json::json!({
|
||||||
|
"url": url
|
||||||
|
}),
|
||||||
|
Arc::new(MemoryState::new("")),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(res.contains("HTTP Error: 404 Not Found"));
|
||||||
|
assert!(res.contains("Not Found Error"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -331,6 +331,10 @@ impl McpTool for QueryRecentChangesHandler {
|
|||||||
.skip(offset)
|
.skip(offset)
|
||||||
.take(limit)
|
.take(limit)
|
||||||
.cloned()
|
.cloned()
|
||||||
|
.map(|mut c| {
|
||||||
|
c.file_path = crate::handlers::utils::sanitize_path(&c.file_path);
|
||||||
|
c
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
Ok::<String, crate::error::AppError>(serde_yaml::to_string(&filtered)?)
|
Ok::<String, crate::error::AppError>(serde_yaml::to_string(&filtered)?)
|
||||||
})?;
|
})?;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
pub mod ast;
|
pub mod ast;
|
||||||
|
pub mod http;
|
||||||
pub mod env;
|
pub mod env;
|
||||||
pub mod git;
|
pub mod git;
|
||||||
pub mod graph;
|
pub mod graph;
|
||||||
|
|||||||
@@ -15,7 +15,8 @@ pub fn now_secs() -> u64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn validate_safe_path(path_str: &str) -> crate::error::Result<std::path::PathBuf> {
|
pub fn validate_safe_path(path_str: &str) -> crate::error::Result<std::path::PathBuf> {
|
||||||
let path = std::path::Path::new(path_str);
|
let sanitized_str = sanitize_path(path_str);
|
||||||
|
let path = std::path::Path::new(&sanitized_str);
|
||||||
for component in path.components() {
|
for component in path.components() {
|
||||||
if component == std::path::Component::ParentDir {
|
if component == std::path::Component::ParentDir {
|
||||||
return Err(crate::error::AppError::BadRequest(format!(
|
return Err(crate::error::AppError::BadRequest(format!(
|
||||||
@@ -27,6 +28,35 @@ pub fn validate_safe_path(path_str: &str) -> crate::error::Result<std::path::Pat
|
|||||||
Ok(path.to_path_buf())
|
Ok(path.to_path_buf())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sanitizes a given path based on the operating environment of the server.
|
||||||
|
/// This translates Windows paths to WSL paths if the server runs in Linux,
|
||||||
|
/// and translates WSL paths to Windows paths if the server runs in Windows.
|
||||||
|
pub fn sanitize_path(raw_path: &str) -> String {
|
||||||
|
if cfg!(unix) {
|
||||||
|
if raw_path.contains(":\\") {
|
||||||
|
if let Ok(output) = std::process::Command::new("wslpath").arg("-u").arg(raw_path).output() {
|
||||||
|
if output.status.success() {
|
||||||
|
return String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if raw_path.len() >= 3 && &raw_path[1..3] == ":\\" {
|
||||||
|
let drive_letter = &raw_path[0..1].to_lowercase();
|
||||||
|
let remainder = raw_path[3..].replace("\\", "/");
|
||||||
|
return format!("/mnt/{}/{}", drive_letter, remainder);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if cfg!(windows) {
|
||||||
|
if raw_path.starts_with("/mnt/") && raw_path.len() > 6 {
|
||||||
|
let drive_letter = &raw_path[5..6].to_uppercase();
|
||||||
|
let remainder = raw_path[7..].replace("/", "\\");
|
||||||
|
return format!("{}:\\{}", drive_letter, remainder);
|
||||||
|
} else if raw_path.starts_with("/") {
|
||||||
|
return format!("\\\\wsl$\\Ubuntu{}", raw_path.replace("/", "\\"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
raw_path.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -536,6 +536,7 @@ impl MemoryHandler {
|
|||||||
register!(graph::SummarizeSubgraphHandler);
|
register!(graph::SummarizeSubgraphHandler);
|
||||||
register!(meta::AgentSignalsHandler);
|
register!(meta::AgentSignalsHandler);
|
||||||
register!(meta::AutoSessionCheckpointHandler);
|
register!(meta::AutoSessionCheckpointHandler);
|
||||||
|
register!(http::FetchUrlHandler);
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
state,
|
state,
|
||||||
|
|||||||
@@ -1033,3 +1033,12 @@ pub struct ClipboardTool {
|
|||||||
/// Image path (for 'write').
|
/// Image path (for 'write').
|
||||||
pub image_path: Option<String>,
|
pub image_path: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fetch content from a URL via an HTTP GET request natively (supports proxy config via environment variables: HTTP_PROXY, HTTPS_PROXY, NO_PROXY).
|
||||||
|
#[derive(Debug, Deserialize, Serialize, JsonSchema)]
|
||||||
|
pub struct FetchUrlTool {
|
||||||
|
/// URL to fetch content from.
|
||||||
|
pub url: String,
|
||||||
|
/// Optional custom user-agent string.
|
||||||
|
pub user_agent: Option<String>,
|
||||||
|
}
|
||||||
Reference in new issue
Block a user