Fix Git source sync failures

This commit is contained in:
george
2026-06-06 00:48:21 +01:00
parent 54986cda99
commit fc683b2803
4 changed files with 111 additions and 38 deletions
+26 -29
View File
@@ -1,6 +1,7 @@
# Git Source Operations for Repository Cloning and File Discovery
import os
import shutil
from subprocess import run
from pathlib import Path
from typing import List, Optional, Dict, Any
@@ -27,6 +28,13 @@ class GitCloneError(Exception):
pass
def run_git(command: List[str]) -> None:
result = run(command, capture_output=True, text=True)
if result.returncode != 0:
error = (result.stderr or result.stdout or "unknown git error").strip()
raise GitCloneError(error)
def clone_or_update_repo(
repo_id: str,
repo_url: str,
@@ -55,37 +63,26 @@ def clone_or_update_repo(
if repo_path.exists():
# Update existing clone
print(f" [Git] Updating existing clone at {repo_path}")
from subprocess import run, CalledProcessError
import subprocess
# Fetch latest changes
result = run(
["git", "-C", str(repo_path), "fetch", "origin"],
capture_output=True,
text=True
)
if result.returncode != 0:
raise GitCloneError(f"Failed to fetch: {result.stderr}")
run_git(["git", "-C", str(repo_path), "fetch", "origin"])
# Reset to branch
run(
["git", "-C", str(repo_path), "reset", "--hard", "origin/" + branch],
capture_output=True,
text=True
)
run_git(["git", "-C", str(repo_path), "reset", "--hard", "origin/" + branch])
else:
# Clone new repository
print(f" [Git] Cloning {repo_url} to {repo_path}")
run(
["git", "-C", str(repo_path.parent), "clone",
"--branch", branch,
"--single-branch",
repo_url, "."],
capture_output=True,
text=True
run_git(
[
"git",
"clone",
"--branch",
branch,
"--single-branch",
repo_url,
str(repo_path),
]
)
print(f" [Git] Checked out branch: {branch}")
@@ -97,8 +94,8 @@ def clone_or_update_repo(
"branch": branch
}
except CalledProcessError as e:
raise GitCloneError(f"Git command failed: {e.stderr}") from e
except GitCloneError:
raise
except Exception as e:
raise GitCloneError(f"Failed to clone/update repo: {e}") from e
@@ -386,4 +383,4 @@ async def sync_sources(
results.append(result)
return results
return results
+18 -9
View File
@@ -432,15 +432,24 @@ async def sync_sources_api(payload: Optional[SyncSourcesRequest] = None):
results = []
for source in sources:
result = await ingest_git_source(
library_id=source["library_id"],
name=source.get("name") or source["library_id"],
description=source.get("description"),
repo_url=source["repo_url"],
branch=source.get("branch", "main"),
include_paths=source.get("include_paths"),
exclude_paths=source.get("exclude_paths"),
)
library_id = source.get("library_id", "unknown")
try:
result = await ingest_git_source(
library_id=library_id,
name=source.get("name") or library_id,
description=source.get("description"),
repo_url=source["repo_url"],
branch=source.get("branch", "main"),
include_paths=source.get("include_paths"),
exclude_paths=source.get("exclude_paths"),
)
except Exception as exc:
result = {
"success": False,
"library_id": library_id,
"repo_url": source.get("repo_url"),
"error": str(exc),
}
results.append(result)
successful = len([r for r in results if r.get("success")])