perf: completely remove disk I/O from memory access paths, optimize Tantivy indexing, and fix MemoryIndex commits

server/src/main.rs

@@ -123,37 +123,46 @@ async fn git_sync_worker(state: Arc<MemoryState>) {
     loop {
         tokio::time::sleep(tokio::time::Duration::from_secs(30)).await;
         
-        if let Ok(repo) = git2::Repository::discover(&repo_path) {
-            if let Ok(head) = repo.head() {
-                if let Ok(commit) = head.peel_to_commit() {
-                    let current_id = commit.id().to_string();
-                    if current_id != last_commit_id && !last_commit_id.is_empty() {
+        let repo_path_clone = repo_path.clone();
+        let commit_data = tokio::task::spawn_blocking(move || {
+            if let Ok(repo) = git2::Repository::discover(&repo_path_clone) {
+                if let Ok(head) = repo.head() {
+                    if let Ok(commit) = head.peel_to_commit() {
+                        let current_id = commit.id().to_string();
                         let msg = commit.message().unwrap_or("").to_string();
                         let branch = head.shorthand().unwrap_or("unknown").to_string();
-                        
-                        state.ledger.modify(|changes| {
-                            changes.push(crate::models::CodeChange {
-                                git_commit: Some(current_id.clone()),
-                                git_branch: Some(branch),
-                                description: format!("Auto-synced commit: {}", msg.trim()),
-                                timestamp: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs(),
-                                file_path: "".to_string(),
-                            });
-                        });
-                        eprintln!("Git Sync: Logged new commit {}", current_id);
-                        
-                        state.tasks.modify(|tasks| {
-                            for task in tasks.iter_mut() {
-                                if task.status != "completed" && msg.to_lowercase().contains(&task.title.to_lowercase()) {
-                                    task.status = "completed".to_string();
-                                    eprintln!("Git Sync: Auto-completed task '{}'", task.title);
-                                }
-                            }
-                        });
+                        return Some((current_id, msg, branch));
                     }
-                    last_commit_id = current_id;
                 }
             }
+            None
+        })
+        .await
+        .unwrap_or(None);
+
+        if let Some((current_id, msg, branch)) = commit_data {
+            if current_id != last_commit_id && !last_commit_id.is_empty() {
+                state.ledger.modify(|changes| {
+                    changes.push(crate::models::CodeChange {
+                        git_commit: Some(current_id.clone()),
+                        git_branch: Some(branch),
+                        description: format!("Auto-synced commit: {}", msg.trim()),
+                        timestamp: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs(),
+                        file_path: "".to_string(),
+                    });
+                });
+                eprintln!("Git Sync: Logged new commit {}", current_id);
+                
+                state.tasks.modify(|tasks| {
+                    for task in tasks.iter_mut() {
+                        if task.status != "completed" && msg.to_lowercase().contains(&task.title.to_lowercase()) {
+                            task.status = "completed".to_string();
+                            eprintln!("Git Sync: Auto-completed task '{}'", task.title);
+                        }
+                    }
+                });
+            }
+            last_commit_id = current_id;
         }
     }
 }
@@ -161,12 +170,20 @@ async fn git_sync_worker(state: Arc<MemoryState>) {
 async fn reconcile_worker(state: Arc<MemoryState>) {
     loop {
         sleep(Duration::from_secs(5)).await;
-        let pattern = format!("{}/delta_*.json", state.base_dir.display());
+        
         let has_local = {
             let session = state.session_graph.read().unwrap();
             !session.entities.is_empty() || !session.relations.is_empty()
         };
-        let has_files = glob::glob(&pattern).map(|p| p.count() > 0).unwrap_or(false);
+
+        let base_dir = state.base_dir.clone();
+        let has_files = tokio::task::spawn_blocking(move || {
+            let pattern = format!("{}/delta_*.json", base_dir.display());
+            glob::glob(&pattern).map(|p| p.count() > 0).unwrap_or(false)
+        })
+        .await
+        .unwrap_or(false);
+
         if has_local || has_files {
             state.apply_sync_write(|_master| {}).await;
             let state_clone = state.clone();
@@ -174,7 +191,6 @@ async fn reconcile_worker(state: Arc<MemoryState>) {
                 state_clone.rebuild_index();
             }).await;
         }
-
     }
 }
 
@@ -754,6 +770,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         context_workspaces: Store::new("context_workspaces", db.clone()),
     });
 
+    state.recover_wal();
     state.rebuild_index();
 
     run_server(state)