feat(server): refactor handlers, router, state management, and memory tools
This commit is contained in:
1 parent
87ddb01063
commit
a083719cf1
36 files changed
+1899
-597
No files matched your search
+98
-11
@@ -1,6 +1,6 @@
|
||||
use crate::state::MemoryState;
|
||||
use crate::models::Snippet;
|
||||
use crate::embedding::generate_embeddings_async;
|
||||
use crate::models::Snippet;
|
||||
use crate::state::MemoryState;
|
||||
use ignore::WalkBuilder;
|
||||
use std::sync::Arc;
|
||||
use tree_sitter::{Node, Parser};
|
||||
@@ -25,7 +25,11 @@ pub async fn start_background_indexer(state: Arc<MemoryState>) {
|
||||
if entry.file_type().is_some_and(|ft| ft.is_file()) {
|
||||
let path = entry.path().to_path_buf();
|
||||
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
|
||||
if ["rs", "ts", "js", "jsx", "tsx", "py", "java", "c", "cpp", "go"].contains(&ext) {
|
||||
if [
|
||||
"rs", "ts", "js", "jsx", "tsx", "py", "java", "c", "cpp", "go",
|
||||
]
|
||||
.contains(&ext)
|
||||
{
|
||||
files_to_process.push(path);
|
||||
}
|
||||
}
|
||||
@@ -65,10 +69,11 @@ pub async fn start_background_indexer(state: Arc<MemoryState>) {
|
||||
// Generate embedding
|
||||
if let Ok(mut emb) = generate_embeddings_async(vec![code.clone()]).await {
|
||||
let embedding = emb.pop();
|
||||
|
||||
let file_name = file_path.file_name().unwrap_or_default().to_string_lossy();
|
||||
|
||||
let file_name =
|
||||
file_path.file_name().unwrap_or_default().to_string_lossy();
|
||||
let snippet_name = format!("{}:{}", file_name, name);
|
||||
|
||||
|
||||
let snippet = Snippet {
|
||||
name: snippet_name.to_string(),
|
||||
language: ext.to_string(),
|
||||
@@ -98,8 +103,34 @@ pub async fn start_background_indexer(state: Arc<MemoryState>) {
|
||||
});
|
||||
}
|
||||
|
||||
fn extract_chunks(node: Node, code: &str, chunks: &mut Vec<(String, String, String)>, _ext: &str) {
|
||||
fn extract_chunks(node: Node, code: &str, chunks: &mut Vec<(String, String, String)>, ext: &str) {
|
||||
extract_chunks_with_parent(node, code, chunks, ext, None);
|
||||
}
|
||||
|
||||
fn extract_chunks_with_parent(
|
||||
node: Node,
|
||||
code: &str,
|
||||
chunks: &mut Vec<(String, String, String)>,
|
||||
ext: &str,
|
||||
parent_scope: Option<&str>,
|
||||
) {
|
||||
let kind = node.kind();
|
||||
let is_impl_or_class = matches!(kind, "impl_item" | "class_declaration" | "class_definition");
|
||||
|
||||
let current_scope = if is_impl_or_class {
|
||||
let mut cursor = node.walk();
|
||||
let mut type_name = None;
|
||||
for child in node.children(&mut cursor) {
|
||||
if child.kind() == "type_identifier" || child.kind() == "name" || child.kind() == "identifier" {
|
||||
type_name = child.utf8_text(code.as_bytes()).ok().map(|s| s.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
type_name.or_else(|| parent_scope.map(|s| s.to_string()))
|
||||
} else {
|
||||
parent_scope.map(|s| s.to_string())
|
||||
};
|
||||
|
||||
let is_structural = matches!(
|
||||
kind,
|
||||
"function_item"
|
||||
@@ -111,24 +142,80 @@ fn extract_chunks(node: Node, code: &str, chunks: &mut Vec<(String, String, Stri
|
||||
);
|
||||
|
||||
if is_structural {
|
||||
let code_text = node.utf8_text(code.as_bytes()).unwrap_or("").to_string();
|
||||
let mut raw_text = node.utf8_text(code.as_bytes()).unwrap_or("").to_string();
|
||||
|
||||
let mut name = "unknown".to_string();
|
||||
let mut cursor = node.walk();
|
||||
for child in node.children(&mut cursor) {
|
||||
let child_kind = child.kind();
|
||||
if child_kind == "identifier" || child_kind == "name" || child_kind == "type_identifier" {
|
||||
name = child.utf8_text(code.as_bytes()).unwrap_or("unknown").to_string();
|
||||
name = child
|
||||
.utf8_text(code.as_bytes())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref scope) = current_scope {
|
||||
raw_text = format!("// Parent Scope: {}\n{}", scope, raw_text);
|
||||
name = format!("{}::{}", scope, name);
|
||||
}
|
||||
|
||||
let desc = format!("{} AST node", kind);
|
||||
chunks.push((name, code_text, desc));
|
||||
chunks.push((name, raw_text, desc));
|
||||
} else {
|
||||
let mut cursor = node.walk();
|
||||
for child in node.named_children(&mut cursor) {
|
||||
extract_chunks(child, code, chunks, _ext);
|
||||
extract_chunks_with_parent(child, code, chunks, ext, current_scope.as_deref());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tree_sitter::Parser;
|
||||
|
||||
#[test]
|
||||
fn test_extract_chunks_rust_function() {
|
||||
let code = "fn test_func() { println!(\"hello\"); }";
|
||||
let mut parser = Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_rust::LANGUAGE.into())
|
||||
.unwrap();
|
||||
let tree = parser.parse(code, None).unwrap();
|
||||
|
||||
let mut chunks = Vec::new();
|
||||
extract_chunks(tree.root_node(), code, &mut chunks, "rs");
|
||||
|
||||
assert_eq!(chunks.len(), 1);
|
||||
assert_eq!(chunks[0].0, "test_func");
|
||||
assert!(chunks[0].1.contains("println"));
|
||||
assert_eq!(chunks[0].2, "function_item AST node");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_chunks_python_function() {
|
||||
let code = "def my_python_func():\n pass\n";
|
||||
let mut parser = Parser::new();
|
||||
parser
|
||||
.set_language(&tree_sitter_python::LANGUAGE.into())
|
||||
.unwrap();
|
||||
let tree = parser.parse(code, None).unwrap();
|
||||
|
||||
let mut chunks = Vec::new();
|
||||
extract_chunks(tree.root_node(), code, &mut chunks, "py");
|
||||
|
||||
assert_eq!(chunks.len(), 1);
|
||||
assert_eq!(chunks[0].0, "my_python_func");
|
||||
assert_eq!(chunks[0].2, "function_definition AST node");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_start_background_indexer_lifecycle() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let state = Arc::new(MemoryState::new(temp_dir.path().to_str().unwrap()));
|
||||
start_background_indexer(state).await;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user