Fix git sync and add repo browser with path selection

- Fix discover_files: rel_path always computed (was stuck at '.' at root), include_path_match now uses relative path, 'return' changed to 'continue' - Fix ingest_git_source: files were cloned but ingested from wrong path (docs/repo-id instead of data/repos/repo-id). Now stages filtered files into DOCS_PATH/library_id before calling ingest_library. - Add browse_repo_tree() for interactive repo exploration - Add POST /api/v1/sources/browse endpoint to backend - Add /sources/browse proxy route to webui - Rewrite sources.html: browse repo, expand/collapse tree, check paths to include, then save source and sync Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-06 01:28:10 +01:00
parent 1b61af8873
commit ff4da0cb9e
4 changed files with 394 additions and 128 deletions
@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import List, Optional

 from fastapi import FastAPI, File, Form, Request, UploadFile
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates

@@ -257,6 +257,19 @@ def parse_path_list(value: str) -> List[str]:
    return paths


+@app.post("/sources/browse")
+async def browse_source(repo_url: str = Form(...), branch: str = Form("main")):
+    client = get_client()
+    try:
+        result = await client.post(
+            "/api/v1/sources/browse",
+            json={"repo_url": repo_url, "branch": branch},
+        )
+        return JSONResponse(content=result)
+    except Exception as e:
+        return JSONResponse(status_code=400, content={"success": False, "error": str(e)})
+
+
@app.post("/sources/add")
 async def add_source(
    library_id: str = Form(...),
@@ -3,59 +3,267 @@
 {% block title %}Sources - Context7 Docs{% endblock %}

 {% block content %}
-<h2>Git Repository Sync</h2>
+<h2>Git Repository Sources</h2>

-<div class="status-message">Add Git repositories to <code>docs_sources.yaml</code>, then sync them into searchable libraries.</div>
+<div class="status-message">Add a Git repository, browse its structure, select paths to include, then save and sync.</div>

-<h3>Add Git Source</h3>
-<form method="post" action="/sources/add" class="sync-form">
-    <label for="library_id">Library ID:</label>
-    <input type="text" id="library_id" name="library_id" placeholder="my-project" required>
+<!-- Step 1: Enter repo details and browse -->
+<div class="create-form">
+  <h3>Add Git Source</h3>

-    <label for="repo_url">Repository URL:</label>
-    <input type="text" id="repo_url" name="repo_url" placeholder="https://github.com/user/repo.git" required>
-
-    <label for="name">Display Name:</label>
-    <input type="text" id="name" name="name" placeholder="My Project">
-
-    <label for="description">Description:</label>
-    <input type="text" id="description" name="description" placeholder="Project documentation">
-
-    <label for="branch">Branch:</label>
-    <input type="text" id="branch" name="branch" value="main">
-
-    <label for="include_paths">Include Paths:</label>
-    <textarea id="include_paths" name="include_paths" rows="4">docs</textarea>
-
-    <label for="exclude_paths">Exclude Paths:</label>
-    <textarea id="exclude_paths" name="exclude_paths" rows="4">node_modules
-.git</textarea>
-
-    <button type="submit">Save Git Source</button>
-</form>
-
-<form method="post" action="/sources/sync" class="sync-form">
-    <label for="override">Override existing repos:</label>
-    <input type="checkbox" id="override" name="override">
-    <button type="submit">Sync All Repositories</button>
-</form>
-
-<div id="source-list"></div>
-
-{% if sources %}
-<h3>Configured Sources</h3>
-<div class="source-cards">
-    {% for src in sources %}
-    <div class="source-card">
-        <strong>{{ src.library_id | default('unknown') }}</strong><br>
-        URL: {{ src.repo_url | default('N/A') }}<br>
-        Branch: {{ src.branch | default('main') }}<br>
-        Include: {{ src.include_paths | default(['*']) | join(', ') }}
+  <div style="display:flex;gap:10px;flex-wrap:wrap;margin-bottom:8px;">
+    <div style="flex:2;min-width:220px;">
+      <label for="browse_url" style="display:block;font-weight:bold;margin-bottom:4px;">Repository URL</label>
+      <input type="text" id="browse_url" placeholder="https://github.com/user/repo.git"
+             style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #ccc;border-radius:4px;">
    </div>
-    {% endfor %}
+    <div style="flex:0 0 120px;">
+      <label for="browse_branch" style="display:block;font-weight:bold;margin-bottom:4px;">Branch</label>
+      <input type="text" id="browse_branch" value="main"
+             style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #ccc;border-radius:4px;">
+    </div>
+    <div style="flex:0 0 auto;align-self:flex-end;padding-bottom:1px;">
+      <button type="button" id="browse-btn" class="btn btn-secondary" onclick="browseRepo()">Browse Repository</button>
+    </div>
+  </div>
+
+  <!-- Tree viewer -->
+  <div id="tree-section" style="display:none;margin-top:12px;">
+    <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:6px;">
+      <strong>Select paths to include:</strong>
+      <span>
+        <button type="button" class="btn btn-sm btn-info" onclick="selectAll()">Select All Dirs</button>
+        <button type="button" class="btn btn-sm" style="background:#888;color:#fff;" onclick="clearAll()">Clear</button>
+      </span>
+    </div>
+    <div id="tree-container"
+         style="border:1px solid #ccc;border-radius:4px;padding:10px;max-height:380px;overflow-y:auto;background:#fafafa;font-family:monospace;font-size:0.88rem;">
+    </div>
+  </div>
+
+  <!-- Step 2: Library details (shown after browse) -->
+  <div id="save-section" style="display:none;margin-top:16px;">
+    <form method="post" action="/sources/add" id="save-form">
+      <input type="hidden" id="include_paths" name="include_paths" value="">
+
+      <div style="display:grid;grid-template-columns:1fr 1fr;gap:10px;margin-bottom:10px;">
+        <div>
+          <label style="display:block;font-weight:bold;margin-bottom:4px;">Library ID <span style="color:red;">*</span></label>
+          <input type="text" name="library_id" id="lib_id" placeholder="my-project" required
+                 style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #ccc;border-radius:4px;">
+        </div>
+        <div>
+          <label style="display:block;font-weight:bold;margin-bottom:4px;">Display Name</label>
+          <input type="text" name="name" id="lib_name" placeholder="My Project"
+                 style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #ccc;border-radius:4px;">
+        </div>
+        <div>
+          <label style="display:block;font-weight:bold;margin-bottom:4px;">Description</label>
+          <input type="text" name="description" placeholder="Optional description"
+                 style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #ccc;border-radius:4px;">
+        </div>
+        <div>
+          <label style="display:block;font-weight:bold;margin-bottom:4px;">Exclude Paths</label>
+          <input type="text" name="exclude_paths" value="node_modules,.git" placeholder="node_modules,.git"
+                 style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #ccc;border-radius:4px;">
+        </div>
+      </div>
+
+      <input type="hidden" name="repo_url" id="form_repo_url">
+      <input type="hidden" name="branch" id="form_branch">
+
+      <div id="selected-paths-preview"
+           style="background:#e8f4fd;border-radius:4px;padding:8px;margin-bottom:10px;font-size:0.85rem;display:none;">
+        <strong>Selected include paths:</strong> <span id="paths-list"></span>
+      </div>
+
+      <button type="submit" class="btn btn-primary">Save Git Source</button>
+    </form>
+  </div>
+</div>
+
+<!-- Configured sources list -->
+{% if sources %}
+<h3 style="margin-top:30px;">Configured Sources ({{ sources|length }})</h3>
+<div class="source-cards">
+  {% for src in sources %}
+  <div class="source-card">
+    <strong>{{ src.library_id | default('unknown') }}</strong>
+    &nbsp;<span style="color:#666;font-size:0.85rem;">{{ src.repo_url | default('') }}</span><br>
+    Branch: {{ src.branch | default('main') }}
+    &nbsp;|&nbsp; Include: {{ src.include_paths | default(['*']) | join(', ') }}
+    {% if src.exclude_paths %}
+    &nbsp;|&nbsp; Exclude: {{ src.exclude_paths | join(', ') }}
+    {% endif %}
+  </div>
+  {% endfor %}
 </div>
 {% else %}
-<p>No git sources configured. Add repositories to <code>docs_sources.yaml</code>.</p>
+<p style="margin-top:20px;color:#666;">No git sources configured yet.</p>
 {% endif %}

+<div style="margin-top:24px;">
+  <form method="post" action="/sources/sync" style="display:inline;">
+    <button type="submit" class="btn btn-primary">Sync All Repositories</button>
+  </form>
+</div>
+
+{% endblock %}
+
+{% block scripts %}
+<script>
+const checkedPaths = new Set();
+
+async function browseRepo() {
+  const url = document.getElementById('browse_url').value.trim();
+  const branch = document.getElementById('browse_branch').value.trim() || 'main';
+  if (!url) { alert('Enter a repository URL first.'); return; }
+
+  const btn = document.getElementById('browse-btn');
+  btn.disabled = true;
+  btn.textContent = 'Cloning…';
+
+  try {
+    const fd = new FormData();
+    fd.append('repo_url', url);
+    fd.append('branch', branch);
+    const resp = await fetch('/sources/browse', { method: 'POST', body: fd });
+    const data = await resp.json();
+
+    if (!data.success) {
+      alert('Browse failed: ' + (data.detail || data.error || 'unknown error'));
+      return;
+    }
+
+    checkedPaths.clear();
+    renderTree(data.tree);
+    document.getElementById('tree-section').style.display = 'block';
+    document.getElementById('save-section').style.display = 'block';
+    document.getElementById('form_repo_url').value = url;
+    document.getElementById('form_branch').value = branch;
+
+    // Auto-fill library ID from repo name
+    const repoName = url.split('/').pop().replace(/\.git$/, '').toLowerCase().replace(/[^a-z0-9-]/g, '-');
+    if (!document.getElementById('lib_id').value) {
+      document.getElementById('lib_id').value = repoName;
+      document.getElementById('lib_name').value = repoName;
+    }
+  } catch(e) {
+    alert('Request failed: ' + e.message);
+  } finally {
+    btn.disabled = false;
+    btn.textContent = 'Browse Repository';
+  }
+}
+
+function renderTree(nodes) {
+  const container = document.getElementById('tree-container');
+  container.innerHTML = buildTreeHTML(nodes, 0);
+}
+
+function buildTreeHTML(nodes, depth) {
+  if (!nodes || nodes.length === 0) return '';
+  let html = '<ul style="list-style:none;padding-left:' + (depth === 0 ? 0 : 18) + 'px;margin:0;">';
+  for (const node of nodes) {
+    if (node.type === 'dir') {
+      const hasChildren = node.children && node.children.length > 0;
+      html += `<li style="margin:2px 0;">
+        <span class="tree-toggle" onclick="toggleDir(this)" style="cursor:pointer;user-select:none;">&#9656;</span>
+        <label style="cursor:pointer;">
+          <input type="checkbox" class="dir-check" data-path="${node.path}"
+                 onchange="onDirCheck(this)"> &#128193; ${node.path.split('/').pop()}/
+        </label>
+        <div class="tree-children" style="display:none;">
+          ${hasChildren ? buildTreeHTML(node.children, depth + 1) : ''}
+        </div>
+      </li>`;
+    } else {
+      html += `<li style="margin:2px 0;color:#555;">
+        <label style="cursor:pointer;padding-left:20px;">
+          <input type="checkbox" class="file-check" data-path="${node.path}"
+                 onchange="onFileCheck(this)"> &#128196; ${node.path.split('/').pop()}
+        </label>
+      </li>`;
+    }
+  }
+  html += '</ul>';
+  return html;
+}
+
+function toggleDir(arrow) {
+  const li = arrow.closest('li');
+  const children = li.querySelector('.tree-children');
+  if (children.style.display === 'none') {
+    children.style.display = 'block';
+    arrow.innerHTML = '&#9662;';
+  } else {
+    children.style.display = 'none';
+    arrow.innerHTML = '&#9656;';
+  }
+}
+
+function onDirCheck(cb) {
+  if (cb.checked) {
+    checkedPaths.add(cb.dataset.path);
+    // Uncheck any parent dirs that are already checked (this dir is more specific)
+    // and uncheck descendant checkboxes to avoid redundancy
+    const li = cb.closest('li');
+    li.querySelectorAll('.dir-check,.file-check').forEach(c => {
+      if (c !== cb) { c.checked = false; checkedPaths.delete(c.dataset.path); }
+    });
+  } else {
+    checkedPaths.delete(cb.dataset.path);
+  }
+  updatePreview();
+}
+
+function onFileCheck(cb) {
+  if (cb.checked) checkedPaths.add(cb.dataset.path);
+  else checkedPaths.delete(cb.dataset.path);
+  updatePreview();
+}
+
+function selectAll() {
+  checkedPaths.clear();
+  document.querySelectorAll('.dir-check').forEach(cb => {
+    // Only select top-level dirs
+    if (!cb.dataset.path.includes('/')) {
+      cb.checked = true;
+      checkedPaths.add(cb.dataset.path);
+      cb.closest('li').querySelectorAll('.dir-check,.file-check').forEach(c => {
+        if (c !== cb) { c.checked = false; }
+      });
+    }
+  });
+  updatePreview();
+}
+
+function clearAll() {
+  checkedPaths.clear();
+  document.querySelectorAll('.dir-check,.file-check').forEach(c => c.checked = false);
+  updatePreview();
+}
+
+function updatePreview() {
+  const paths = [...checkedPaths].sort();
+  document.getElementById('include_paths').value = paths.join('\n');
+  const preview = document.getElementById('selected-paths-preview');
+  const list = document.getElementById('paths-list');
+  if (paths.length > 0) {
+    preview.style.display = 'block';
+    list.textContent = paths.join(', ');
+  } else {
+    preview.style.display = 'none';
+    list.textContent = '';
+  }
+}
+
+document.getElementById('save-form').addEventListener('submit', function(e) {
+  if (checkedPaths.size === 0) {
+    if (!confirm('No paths selected — this will ingest the entire repository. Continue?')) {
+      e.preventDefault();
+    }
+  }
+});
+</script>
 {% endblock %}