george/DocsMCP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve ingestion data across rebuilds

Browse Source
This commit is contained in:
george
2026-06-06 12:44:02 +01:00
parent f3509a363e
commit 7707a6306d
6 changed files with 194 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/db.py
+51
View File
@@ -245,6 +245,57 @@ def clear_library_documents(library_id: str) -> Dict[str, Any]:
conn.close()
def replace_library_documents(
library_id: str,
chunks: List[Dict[str, Any]],
) -> Dict[str, Any]:
"""Atomically replace all document chunks for a library."""
conn = get_connection()
try:
now = datetime.utcnow().isoformat()
conn.execute("BEGIN")
cursor = conn.execute(
"DELETE FROM documents WHERE library_id = ?", (library_id,)
)
deleted = cursor.rowcount
conn.executemany(
"""
INSERT INTO documents
(id, library_id, path, title, content, chunk_index,
token_estimate, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
[
(
chunk["id"],
library_id,
chunk["path"],
chunk.get("title"),
chunk.get("content"),
chunk.get("chunk_index"),
chunk.get("token_estimate", 0),
now,
)
for chunk in chunks
],
)
conn.commit()
return {
"success": True,
"deleted": deleted,
"inserted": len(chunks),
"library_id": library_id,
}
except Exception as e:
conn.rollback()
return {"success": False, "error": str(e)}
finally:
conn.close()
def delete_library(library_id: str) -> Dict[str, Any]:
"""Delete a library row and its document chunks."""
conn = get_connection()
backend/app/embeddings.py
+7 -2
View File
@@ -1,5 +1,6 @@
# Local Embedding Generation using FastEmbed
import asyncio
import os
from typing import List
from functools import lru_cache
@@ -20,7 +21,11 @@ def _load_model():
print("Loading embedding model (this may take a few minutes on first run)...")
# Use BAAI/bge-small-en-v1.5 - lightweight (~90MB), works offline
_embedding_model = TextEmbedding(model_name="BAAI/bge-small-en-v1.5", cache_dir=".embed_cache")
cache_dir = os.getenv("EMBEDDING_CACHE_DIR", ".embed_cache")
_embedding_model = TextEmbedding(
model_name="BAAI/bge-small-en-v1.5",
cache_dir=cache_dir,
)
print("Embedding model loaded successfully.")
return _embedding_model
@@ -178,4 +183,4 @@ if __name__ == "__main__":
assert embed_texts([]) == [], "Empty list should return empty list"
print("✓ Empty input handling works")
print("\n✅ All tests passed!")
print("\n✅ All tests passed!")
backend/app/ingest.py
+49 -35
View File
@@ -15,7 +15,7 @@ from .config import settings
from .chunking import chunk_text, estimate_tokens
from .embeddings import embed_texts
from .vector_store import upsert_chunks
from .db import insert_document_chunk, upsert_library, clear_library_documents
from .db import replace_library_documents, upsert_library
from .git_source import ingest_git_source
SUPPORTED_EXTENSIONS = {'.md', '.txt', '.py', '.js', '.ts', '.json',
@@ -123,15 +123,7 @@ async def ingest_library(library_id: str, name: str, description: Optional[str]
print(f" [Library] Found {len(doc_files)} document(s)")
# Clear old chunks for this library
print(f" [Library] Clearing existing chunks...")
clear_result = clear_library_documents(library_id)
if not clear_result.get('success'):
print(f" Warning: Could not clear library docs: {clear_result}")
else:
print(f" [Library] Cleared {clear_result.get('deleted', 0)} existing chunks")
# Process documents
# Prepare the complete replacement before touching the existing index.
all_chunks = []
processed_files = 0
@@ -158,56 +150,78 @@ async def ingest_library(library_id: str, name: str, description: Optional[str]
embeddings = await asyncio.to_thread(embed_texts, chunks)
# Build chunk dicts
chunk_dicts = []
base_path = file_path.relative_to(library_dir).as_posix()
for i, chunk in enumerate(chunks):
chunk_dict = {
"id": f"{file_path.stem}-{i}",
"id": f"{base_path}:{i}",
"library_id": library_id,
"path": base_path,
"title": Path(base_path).stem,
"content": chunk,
"chunk_index": i,
"embedding": embeddings[i]
"embedding": embeddings[i],
"token_estimate": estimate_tokens(chunk),
}
all_chunks.append(chunk_dict)
processed_files += 1
print(f" [Library] Processed {processed_files} file(s), {len(all_chunks)} total chunks")
# Save chunks to SQLite
if all_chunks:
for chunk in all_chunks:
insert_result = insert_document_chunk(
doc_id=chunk["id"],
library_id=chunk["library_id"],
path=chunk["path"],
title=chunk.get("title"),
content=chunk["content"],
chunk_index=chunk["chunk_index"],
token_estimate=estimate_tokens(chunk["content"])
)
if insert_result.get('success'):
continue
print(f" [Library] Saved {len(all_chunks)} chunks to SQLite")
else:
print(f" [Library] No chunks to save to SQLite")
# Save vectors to Qdrant
if doc_files and not all_chunks:
error = "No document chunks were produced; keeping the existing index"
print(f" [Library] {error}")
return {
"success": False,
"library_id": library_id,
"files_processed": processed_files,
"chunks_created": 0,
"vectors_added": 0,
"error": error,
}
# Update vectors first. If this fails, the previous SQLite index remains usable.
if all_chunks:
upsert_result = await upsert_chunks(all_chunks)
print(f" [Library] Vector store: {upsert_result.get('success', False)} ({upsert_result.get('points_added', 0)} added)")
if not upsert_result.get("success"):
return {
"success": False,
"library_id": library_id,
"files_processed": processed_files,
"chunks_created": len(all_chunks),
"vectors_added": 0,
"error": upsert_result.get("error", "Vector store update failed"),
}
else:
print(f" [Library] No vectors to add to Qdrant")
upsert_result = {"success": True, "points_added": 0}
# Replace SQLite rows in one transaction only after preparation succeeds.
replace_result = replace_library_documents(library_id, all_chunks)
if not replace_result.get("success"):
print(f" [Library] SQLite replacement failed: {replace_result.get('error')}")
return {
"success": False,
"library_id": library_id,
"files_processed": processed_files,
"chunks_created": len(all_chunks),
"vectors_added": upsert_result.get("points_added", 0),
"error": replace_result.get("error", "SQLite replacement failed"),
}
print(
f" [Library] Replaced {replace_result.get('deleted', 0)} old chunks "
f"with {replace_result.get('inserted', 0)} new chunks"
)
return {
"success": True,
"library_id": library_id,
"files_processed": processed_files,
"chunks_created": len(all_chunks),
"vectors_added": upsert_result.get('points_added', 0) if 'upsert_result' in locals() else len(all_chunks)
"vectors_added": upsert_result.get("points_added", 0),
}