Add Git source management to WebUI

backend/app/main.py

@@ -45,6 +45,16 @@ class SyncSourcesRequest(BaseModel):
     override: bool = False
 
 
+class GitSourceRequest(BaseModel):
+    library_id: str = Field(..., min_length=1)
+    repo_url: str = Field(..., min_length=1)
+    name: Optional[str] = None
+    description: Optional[str] = None
+    branch: str = "main"
+    include_paths: Optional[list[str]] = None
+    exclude_paths: Optional[list[str]] = None
+
+
 DOCUMENT_EXTENSIONS = {
     ".md",
     ".txt",
@@ -190,6 +200,40 @@ def sources_config_path() -> Path:
     return Path(__file__).resolve().parents[2] / "docs_sources.yaml"
 
 
+def clean_source_paths(paths: Optional[list[str]]) -> list[str]:
+    cleaned = []
+    for raw_path in paths or []:
+        path = raw_path.strip().strip("/")
+        if not path or path == "." or ".." in Path(path).parts or Path(path).is_absolute():
+            continue
+        cleaned.append(path)
+    return cleaned
+
+
+def load_sources_config() -> dict:
+    path = sources_config_path()
+    if not path.exists():
+        return {"sources": []}
+
+    with path.open() as f:
+        data = yaml.safe_load(f) or {}
+
+    if isinstance(data, list):
+        return {"sources": data}
+    if not isinstance(data, dict):
+        return {"sources": []}
+    sources = data.get("sources", [])
+    data["sources"] = sources if isinstance(sources, list) else []
+    return data
+
+
+def save_sources_config(data: dict) -> None:
+    path = sources_config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as f:
+        yaml.safe_dump(data, f, sort_keys=False)
+
+
 @app.get("/health")
 async def health_check():
     return {"status": "ok", "service": "docs-api"}
@@ -341,18 +385,45 @@ async def api_upload(library_id: str, file: UploadFile = File(...)):
 @app.get("/api/v1/sources")
 @app.get("/sources/config")
 async def api_list_sources():
-    path = sources_config_path()
+    data = load_sources_config()
-    if not path.exists():
+    sources = data["sources"]
-        return {"success": True, "sources": [], "count": 0}
-
-    with path.open() as f:
-        data = yaml.safe_load(f) or {}
-    sources = data.get("sources", data if isinstance(data, list) else [])
-    if not isinstance(sources, list):
-        sources = []
     return {"success": True, "sources": sources, "count": len(sources)}
 
 
+@app.post("/api/v1/sources")
+async def api_add_source(source: GitSourceRequest):
+    library_id = safe_library_id(source.library_id)
+    branch = source.branch.strip() or "main"
+    include_paths = clean_source_paths(source.include_paths)
+    exclude_paths = clean_source_paths(source.exclude_paths) or ["node_modules", ".git"]
+
+    source_entry = {
+        "library_id": library_id,
+        "name": (source.name or library_id).strip(),
+        "description": (source.description or "").strip(),
+        "repo_url": source.repo_url.strip(),
+        "branch": branch,
+        "include_paths": include_paths or ["docs"],
+        "exclude_paths": exclude_paths,
+    }
+
+    data = load_sources_config()
+    sources = data["sources"]
+    existing_index = next(
+        (index for index, item in enumerate(sources) if item.get("library_id") == library_id),
+        None,
+    )
+    if existing_index is None:
+        sources.append(source_entry)
+        created = True
+    else:
+        sources[existing_index] = source_entry
+        created = False
+
+    save_sources_config(data)
+    return {"success": True, "created": created, "source": source_entry}
+
+
 @app.post("/sources/sync")
 async def sync_sources_api(payload: Optional[SyncSourcesRequest] = None):
     source_data = await api_list_sources()

docker-compose.yml

@@ -37,6 +37,7 @@ services:
     volumes:
       - ./docs:/docs
       - ./data:/data
+      - ./docs_sources.yaml:/app/docs_sources.yaml
     depends_on:
       - qdrant
     networks:

webui/app/main.py

@@ -248,6 +248,43 @@ async def sources_page(request: Request):
     return templates.TemplateResponse("sources.html", {"request": request, "sources": sources})
 
 
+def parse_path_list(value: str) -> List[str]:
+    paths = []
+    for line in value.replace(",", "\n").splitlines():
+        item = line.strip()
+        if item:
+            paths.append(item)
+    return paths
+
+
+@app.post("/sources/add")
+async def add_source(
+    library_id: str = Form(...),
+    repo_url: str = Form(...),
+    name: str = Form(""),
+    description: str = Form(""),
+    branch: str = Form("main"),
+    include_paths: str = Form("docs"),
+    exclude_paths: str = Form("node_modules\n.git"),
+):
+    client = get_client()
+    payload = {
+        "library_id": library_id,
+        "repo_url": repo_url,
+        "name": name or None,
+        "description": description or None,
+        "branch": branch or "main",
+        "include_paths": parse_path_list(include_paths),
+        "exclude_paths": parse_path_list(exclude_paths),
+    }
+    try:
+        result = await client.post("/api/v1/sources", json=payload)
+        body = f"<h1>Git Source Saved</h1><pre>{html.escape(str(result))}</pre><a href='/sources'>Back</a>"
+    except Exception as e:
+        body = f"<h1>Save Failed</h1><pre>{html.escape(str(e))}</pre><a href='/sources'>Back</a>"
+    return page("Git Source Saved", body)
+
+
 @app.post("/sources/sync")
 async def sync_sources(override: bool = Form(False)):
     client = get_client()

webui/app/templates/sources.html

@@ -5,7 +5,34 @@
 {% block content %}
 <h2>Git Repository Sync</h2>
 
-<div class="status-message">Syncs all git repositories configured in <code>docs_sources.yaml</code>.</div>
+<div class="status-message">Add Git repositories to <code>docs_sources.yaml</code>, then sync them into searchable libraries.</div>
+
+<h3>Add Git Source</h3>
+<form method="post" action="/sources/add" class="sync-form">
+    <label for="library_id">Library ID:</label>
+    <input type="text" id="library_id" name="library_id" placeholder="my-project" required>
+
+    <label for="repo_url">Repository URL:</label>
+    <input type="text" id="repo_url" name="repo_url" placeholder="https://github.com/user/repo.git" required>
+
+    <label for="name">Display Name:</label>
+    <input type="text" id="name" name="name" placeholder="My Project">
+
+    <label for="description">Description:</label>
+    <input type="text" id="description" name="description" placeholder="Project documentation">
+
+    <label for="branch">Branch:</label>
+    <input type="text" id="branch" name="branch" value="main">
+
+    <label for="include_paths">Include Paths:</label>
+    <textarea id="include_paths" name="include_paths" rows="4">docs</textarea>
+
+    <label for="exclude_paths">Exclude Paths:</label>
+    <textarea id="exclude_paths" name="exclude_paths" rows="4">node_modules
+.git</textarea>
+
+    <button type="submit">Save Git Source</button>
+</form>
 
 <form method="post" action="/sources/sync" class="sync-form">
     <label for="override">Override existing repos:</label>