Fix long ingestion and Qdrant compatibility
This commit is contained in:
@@ -154,7 +154,8 @@ async def ingest_library(library_id: str, name: str, description: Optional[str]
|
|||||||
# Embed chunks and prepare for storage
|
# Embed chunks and prepare for storage
|
||||||
print(f" Chunked into {len(chunks)} pieces (approx. {num_tokens} tokens)")
|
print(f" Chunked into {len(chunks)} pieces (approx. {num_tokens} tokens)")
|
||||||
|
|
||||||
embeddings = embed_texts(chunks)
|
# FastEmbed is synchronous and CPU-heavy; keep it off the API event loop.
|
||||||
|
embeddings = await asyncio.to_thread(embed_texts, chunks)
|
||||||
|
|
||||||
# Build chunk dicts
|
# Build chunk dicts
|
||||||
chunk_dicts = []
|
chunk_dicts = []
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ python-dotenv==1.0.0
|
|||||||
python-multipart==0.0.6
|
python-multipart==0.0.6
|
||||||
|
|
||||||
# Qdrant Vector Store Client
|
# Qdrant Vector Store Client
|
||||||
qdrant-client==1.7.0
|
qdrant-client==1.18.0
|
||||||
|
|
||||||
# Text Processing for token estimation
|
# Text Processing for token estimation
|
||||||
tiktoken==0.7.0
|
tiktoken==0.7.0
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ fastmcp==2.12.5
|
|||||||
httpx>=0.28.1,<1.0
|
httpx>=0.28.1,<1.0
|
||||||
|
|
||||||
# For Qdrant vector store operations
|
# For Qdrant vector store operations
|
||||||
qdrant-client==1.7.0
|
qdrant-client==1.18.0
|
||||||
|
|
||||||
# Text processing for token estimation
|
# Text processing for token estimation
|
||||||
tiktoken==0.7.0
|
tiktoken==0.7.0
|
||||||
|
|||||||
+5
-1
@@ -49,7 +49,11 @@ async def run_sync_job(job_id: str, override: bool) -> None:
|
|||||||
job["status"] = "running"
|
job["status"] = "running"
|
||||||
job["started_at"] = utc_now()
|
job["started_at"] = utc_now()
|
||||||
try:
|
try:
|
||||||
result = await get_client().post("/sources/sync", json={"override": override})
|
result = await get_client().post(
|
||||||
|
"/sources/sync",
|
||||||
|
json={"override": override},
|
||||||
|
timeout=None,
|
||||||
|
)
|
||||||
job["result"] = result
|
job["result"] = result
|
||||||
job["status"] = "succeeded" if result.get("success") else "failed"
|
job["status"] = "succeeded" if result.get("success") else "failed"
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|||||||
Reference in New Issue
Block a user