Add Git source management to WebUI
This commit is contained in:
george
+80
-9
@@ -45,6 +45,16 @@ class SyncSourcesRequest(BaseModel):
|
||||
override: bool = False
|
||||
|
||||
|
||||
class GitSourceRequest(BaseModel):
|
||||
library_id: str = Field(..., min_length=1)
|
||||
repo_url: str = Field(..., min_length=1)
|
||||
name: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
branch: str = "main"
|
||||
include_paths: Optional[list[str]] = None
|
||||
exclude_paths: Optional[list[str]] = None
|
||||
|
||||
|
||||
DOCUMENT_EXTENSIONS = {
|
||||
".md",
|
||||
".txt",
|
||||
@@ -190,6 +200,40 @@ def sources_config_path() -> Path:
|
||||
return Path(__file__).resolve().parents[2] / "docs_sources.yaml"
|
||||
|
||||
|
||||
def clean_source_paths(paths: Optional[list[str]]) -> list[str]:
|
||||
cleaned = []
|
||||
for raw_path in paths or []:
|
||||
path = raw_path.strip().strip("/")
|
||||
if not path or path == "." or ".." in Path(path).parts or Path(path).is_absolute():
|
||||
continue
|
||||
cleaned.append(path)
|
||||
return cleaned
|
||||
|
||||
|
||||
def load_sources_config() -> dict:
|
||||
path = sources_config_path()
|
||||
if not path.exists():
|
||||
return {"sources": []}
|
||||
|
||||
with path.open() as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
|
||||
if isinstance(data, list):
|
||||
return {"sources": data}
|
||||
if not isinstance(data, dict):
|
||||
return {"sources": []}
|
||||
sources = data.get("sources", [])
|
||||
data["sources"] = sources if isinstance(sources, list) else []
|
||||
return data
|
||||
|
||||
|
||||
def save_sources_config(data: dict) -> None:
|
||||
path = sources_config_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w") as f:
|
||||
yaml.safe_dump(data, f, sort_keys=False)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
return {"status": "ok", "service": "docs-api"}
|
||||
@@ -341,18 +385,45 @@ async def api_upload(library_id: str, file: UploadFile = File(...)):
|
||||
@app.get("/api/v1/sources")
|
||||
@app.get("/sources/config")
|
||||
async def api_list_sources():
|
||||
path = sources_config_path()
|
||||
if not path.exists():
|
||||
return {"success": True, "sources": [], "count": 0}
|
||||
|
||||
with path.open() as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
sources = data.get("sources", data if isinstance(data, list) else [])
|
||||
if not isinstance(sources, list):
|
||||
sources = []
|
||||
data = load_sources_config()
|
||||
sources = data["sources"]
|
||||
return {"success": True, "sources": sources, "count": len(sources)}
|
||||
|
||||
|
||||
@app.post("/api/v1/sources")
|
||||
async def api_add_source(source: GitSourceRequest):
|
||||
library_id = safe_library_id(source.library_id)
|
||||
branch = source.branch.strip() or "main"
|
||||
include_paths = clean_source_paths(source.include_paths)
|
||||
exclude_paths = clean_source_paths(source.exclude_paths) or ["node_modules", ".git"]
|
||||
|
||||
source_entry = {
|
||||
"library_id": library_id,
|
||||
"name": (source.name or library_id).strip(),
|
||||
"description": (source.description or "").strip(),
|
||||
"repo_url": source.repo_url.strip(),
|
||||
"branch": branch,
|
||||
"include_paths": include_paths or ["docs"],
|
||||
"exclude_paths": exclude_paths,
|
||||
}
|
||||
|
||||
data = load_sources_config()
|
||||
sources = data["sources"]
|
||||
existing_index = next(
|
||||
(index for index, item in enumerate(sources) if item.get("library_id") == library_id),
|
||||
None,
|
||||
)
|
||||
if existing_index is None:
|
||||
sources.append(source_entry)
|
||||
created = True
|
||||
else:
|
||||
sources[existing_index] = source_entry
|
||||
created = False
|
||||
|
||||
save_sources_config(data)
|
||||
return {"success": True, "created": created, "source": source_entry}
|
||||
|
||||
|
||||
@app.post("/sources/sync")
|
||||
async def sync_sources_api(payload: Optional[SyncSourcesRequest] = None):
|
||||
source_data = await api_list_sources()
|
||||
|
||||
@@ -37,6 +37,7 @@ services:
|
||||
volumes:
|
||||
- ./docs:/docs
|
||||
- ./data:/data
|
||||
- ./docs_sources.yaml:/app/docs_sources.yaml
|
||||
depends_on:
|
||||
- qdrant
|
||||
networks:
|
||||
|
||||
@@ -248,6 +248,43 @@ async def sources_page(request: Request):
|
||||
return templates.TemplateResponse("sources.html", {"request": request, "sources": sources})
|
||||
|
||||
|
||||
def parse_path_list(value: str) -> List[str]:
|
||||
paths = []
|
||||
for line in value.replace(",", "\n").splitlines():
|
||||
item = line.strip()
|
||||
if item:
|
||||
paths.append(item)
|
||||
return paths
|
||||
|
||||
|
||||
@app.post("/sources/add")
|
||||
async def add_source(
|
||||
library_id: str = Form(...),
|
||||
repo_url: str = Form(...),
|
||||
name: str = Form(""),
|
||||
description: str = Form(""),
|
||||
branch: str = Form("main"),
|
||||
include_paths: str = Form("docs"),
|
||||
exclude_paths: str = Form("node_modules\n.git"),
|
||||
):
|
||||
client = get_client()
|
||||
payload = {
|
||||
"library_id": library_id,
|
||||
"repo_url": repo_url,
|
||||
"name": name or None,
|
||||
"description": description or None,
|
||||
"branch": branch or "main",
|
||||
"include_paths": parse_path_list(include_paths),
|
||||
"exclude_paths": parse_path_list(exclude_paths),
|
||||
}
|
||||
try:
|
||||
result = await client.post("/api/v1/sources", json=payload)
|
||||
body = f"<h1>Git Source Saved</h1><pre>{html.escape(str(result))}</pre><a href='/sources'>Back</a>"
|
||||
except Exception as e:
|
||||
body = f"<h1>Save Failed</h1><pre>{html.escape(str(e))}</pre><a href='/sources'>Back</a>"
|
||||
return page("Git Source Saved", body)
|
||||
|
||||
|
||||
@app.post("/sources/sync")
|
||||
async def sync_sources(override: bool = Form(False)):
|
||||
client = get_client()
|
||||
|
||||
@@ -5,7 +5,34 @@
|
||||
{% block content %}
|
||||
<h2>Git Repository Sync</h2>
|
||||
|
||||
<div class="status-message">Syncs all git repositories configured in <code>docs_sources.yaml</code>.</div>
|
||||
<div class="status-message">Add Git repositories to <code>docs_sources.yaml</code>, then sync them into searchable libraries.</div>
|
||||
|
||||
<h3>Add Git Source</h3>
|
||||
<form method="post" action="/sources/add" class="sync-form">
|
||||
<label for="library_id">Library ID:</label>
|
||||
<input type="text" id="library_id" name="library_id" placeholder="my-project" required>
|
||||
|
||||
<label for="repo_url">Repository URL:</label>
|
||||
<input type="text" id="repo_url" name="repo_url" placeholder="https://github.com/user/repo.git" required>
|
||||
|
||||
<label for="name">Display Name:</label>
|
||||
<input type="text" id="name" name="name" placeholder="My Project">
|
||||
|
||||
<label for="description">Description:</label>
|
||||
<input type="text" id="description" name="description" placeholder="Project documentation">
|
||||
|
||||
<label for="branch">Branch:</label>
|
||||
<input type="text" id="branch" name="branch" value="main">
|
||||
|
||||
<label for="include_paths">Include Paths:</label>
|
||||
<textarea id="include_paths" name="include_paths" rows="4">docs</textarea>
|
||||
|
||||
<label for="exclude_paths">Exclude Paths:</label>
|
||||
<textarea id="exclude_paths" name="exclude_paths" rows="4">node_modules
|
||||
.git</textarea>
|
||||
|
||||
<button type="submit">Save Git Source</button>
|
||||
</form>
|
||||
|
||||
<form method="post" action="/sources/sync" class="sync-form">
|
||||
<label for="override">Override existing repos:</label>
|
||||
|
||||
Reference in New Issue
Block a user