Commit e636c018 authored by Mahmoud Aglan's avatar Mahmoud Aglan

jjjjjj

parent 4bc6c1f8
This source diff could not be displayed because it is too large. You can view the blob instead.
"""
Knowledge base management and document upload (supports multiple files at once).
Knowledge base management and document upload.
Full CRUD for knowledge bases AND their individual documents.
"""
from pydantic import BaseModel
......@@ -22,6 +23,15 @@ class CreateKBBody(BaseModel):
description: str = ""
class UpdateKBBody(BaseModel):
name: Optional[str] = None
description: Optional[str] = None
# ═══════════════════════════════════════════════════
# Knowledge Base CRUD
# ═══════════════════════════════════════════════════
@router.get("")
def list_knowledge_bases(user: User = Depends(get_current_user), db: Session = Depends(get_db)):
kbs = db.query(KnowledgeBase).filter(
......@@ -43,22 +53,35 @@ def create_kb(body: CreateKBBody, user: User = Depends(get_current_user), db: Se
@router.get("/{kb_id}")
def get_kb(kb_id: str, user: User = Depends(get_current_user), db: Session = Depends(get_db)):
kb = _get_kb(kb_id, user, db)
docs = db.query(KnowledgeDocument).filter(KnowledgeDocument.knowledge_base_id == kb_id).all()
docs = (
db.query(KnowledgeDocument)
.filter(KnowledgeDocument.knowledge_base_id == kb_id)
.order_by(KnowledgeDocument.created_at.desc())
.all()
)
return {
**_kb_dict(kb),
"documents": [
{
"id": d.id,
"filename": d.filename,
"file_size": d.file_size,
"chunk_count": d.chunk_count,
"created_at": str(d.created_at),
}
for d in docs
],
"documents": [_doc_dict(d) for d in docs],
}
@router.put("/{kb_id}")
def update_kb(
kb_id: str,
body: UpdateKBBody,
user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
kb = _get_kb(kb_id, user, db)
if body.name is not None:
kb.name = body.name
if body.description is not None:
kb.description = body.description
db.commit()
db.refresh(kb)
return _kb_dict(kb)
@router.delete("/{kb_id}")
def delete_kb(kb_id: str, user: User = Depends(get_current_user), db: Session = Depends(get_db)):
kb = _get_kb(kb_id, user, db)
......@@ -72,6 +95,72 @@ def delete_kb(kb_id: str, user: User = Depends(get_current_user), db: Session =
return {"ok": True}
# ═══════════════════════════════════════════════════
# Document Management
# ═══════════════════════════════════════════════════
@router.get("/{kb_id}/documents")
def list_documents(
kb_id: str,
user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
"""List all documents in a knowledge base."""
_get_kb(kb_id, user, db)
docs = (
db.query(KnowledgeDocument)
.filter(KnowledgeDocument.knowledge_base_id == kb_id)
.order_by(KnowledgeDocument.created_at.desc())
.all()
)
return [_doc_dict(d) for d in docs]
@router.delete("/{kb_id}/documents/{doc_id}")
def delete_document(
kb_id: str,
doc_id: str,
user: User = Depends(get_current_user),
db: Session = Depends(get_db),
):
"""Delete a single document from a knowledge base, including its vector chunks."""
kb = _get_kb(kb_id, user, db)
doc = (
db.query(KnowledgeDocument)
.filter(KnowledgeDocument.id == doc_id, KnowledgeDocument.knowledge_base_id == kb_id)
.first()
)
if not doc:
raise HTTPException(404, "Document not found")
# Remove vector chunks from ChromaDB
try:
removed_count = rag_service.delete_document_chunks(kb_id, doc.filename)
except Exception:
removed_count = 0
# Update KB aggregate stats
kb.document_count = max((kb.document_count or 0) - 1, 0)
kb.chunk_count = max((kb.chunk_count or 0) - (doc.chunk_count or 0), 0)
# Estimate character reduction (rough: chunk_count * avg_chunk_size)
estimated_chars = (doc.chunk_count or 0) * 2000
kb.total_characters = max((kb.total_characters or 0) - estimated_chars, 0)
db.delete(doc)
db.commit()
return {
"ok": True,
"chunks_removed": removed_count,
"document_id": doc_id,
"filename": doc.filename,
}
# ═══════════════════════════════════════════════════
# Upload Documents
# ═══════════════════════════════════════════════════
@router.post("/{kb_id}/upload")
async def upload_documents(
kb_id: str,
......@@ -150,6 +239,10 @@ async def upload_documents(
}
# ═══════════════════════════════════════════════════
# Helpers
# ═══════════════════════════════════════════════════
def _get_kb(kb_id: str, user: User, db: Session) -> KnowledgeBase:
kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first()
if not kb:
......@@ -198,10 +291,20 @@ def _kb_dict(kb: KnowledgeBase) -> dict:
return {
"id": kb.id,
"name": kb.name,
"description": kb.description,
"description": kb.description or "",
"document_count": kb.document_count or 0,
"chunk_count": kb.chunk_count or 0,
"total_characters": kb.total_characters or 0,
"estimated_tokens": (kb.total_characters or 0) // 4,
"created_at": str(kb.created_at),
}
def _doc_dict(d: KnowledgeDocument) -> dict:
return {
"id": d.id,
"filename": d.filename,
"file_size": d.file_size or 0,
"chunk_count": d.chunk_count or 0,
"created_at": str(d.created_at),
}
\ No newline at end of file
"""
RAG (Retrieval-Augmented Generation) via ChromaDB.
Each knowledge base maps to a ChromaDB collection.
Supports document-level deletion by filename metadata.
"""
import os
......@@ -39,7 +40,6 @@ def add_documents(
):
col = _chroma_client.get_or_create_collection(name=_col_name(collection_id))
ids = [str(uuid4()) for _ in documents]
# ChromaDB handles batching internally; we chunk to stay under its limits
batch_size = 500
for i in range(0, len(documents), batch_size):
batch_docs = documents[i : i + batch_size]
......@@ -48,6 +48,40 @@ def add_documents(
col.add(documents=batch_docs, ids=batch_ids, metadatas=batch_meta)
def delete_document_chunks(collection_id: str, filename: str) -> int:
"""
Delete all vector chunks belonging to a specific document (by filename).
Returns the number of chunks removed.
"""
try:
col = _chroma_client.get_collection(name=_col_name(collection_id))
except Exception:
return 0
if col.count() == 0:
return 0
# Fetch all chunk IDs that match this filename
try:
results = col.get(
where={"filename": filename},
include=[], # We only need IDs
)
chunk_ids = results.get("ids", [])
if not chunk_ids:
return 0
# Delete in batches (ChromaDB can handle large deletes but let's be safe)
batch_size = 500
for i in range(0, len(chunk_ids), batch_size):
batch = chunk_ids[i : i + batch_size]
col.delete(ids=batch)
return len(chunk_ids)
except Exception:
return 0
def query(collection_id: str, query_text: str, n_results: int = 8) -> str | None:
"""
Return a formatted string of the top matching chunks,
......
......@@ -256,7 +256,6 @@ tar cf "$TARBALL" \
--exclude='.env.local' \
--exclude='.idea' \
--exclude='.vscode' \
--exclude='./main.py' \
--exclude='create-project.ps1' \
--exclude='*.sh' \
.
......
......@@ -6,6 +6,7 @@ import * as streamManager from "./streamManager";
import LoginPage from "./pages/LoginPage";
import ChatPage from "./pages/ChatPage";
import AdminPage from "./pages/AdminPage";
import KnowledgePage from "./pages/KnowledgePage";
import { Flame } from "lucide-react";
export default function App() {
......@@ -58,6 +59,7 @@ export default function App() {
return (
<Routes>
<Route path="/admin" element={<AdminPage />} />
<Route path="/knowledge" element={<KnowledgePage />} />
<Route path="/*" element={<ChatPage />} />
</Routes>
);
......
......@@ -21,6 +21,10 @@ async function request(method, path, token, body) {
return res.json();
}
// ═══════════════════════════════════════════════════
// Auth
// ═══════════════════════════════════════════════════
export const login = (username, password) =>
request("POST", "/auth/login", null, { username, password });
......@@ -29,6 +33,10 @@ export const register = (username, email, password) =>
export const getMe = (token) => request("GET", "/auth/me", token);
// ═══════════════════════════════════════════════════
// Chats
// ═══════════════════════════════════════════════════
export const listChats = (token) => request("GET", "/chats", token);
export const createChat = (token, data = {}) => request("POST", "/chats", token, data);
......@@ -48,10 +56,16 @@ export const getMessages = (token, chatId) =>
export const checkGenerating = (token, chatId) =>
request("GET", `/chats/${chatId}/generating`, token);
// ═══════════════════════════════════════════════════
// Streaming
// ═══════════════════════════════════════════════════
export async function* streamMessage(token, chatId, body, signal) {
const res = await fetch(`${BASE}/chats/${chatId}/messages`, {
method: "POST", headers: headers(token),
body: JSON.stringify(body), signal,
method: "POST",
headers: headers(token),
body: JSON.stringify(body),
signal,
});
if (!res.ok) {
const err = await res.json().catch(() => ({ detail: res.statusText }));
......@@ -69,20 +83,34 @@ export async function* streamMessage(token, chatId, body, signal) {
for (const part of parts) {
const line = part.trim();
if (line.startsWith("data: ")) {
try { yield JSON.parse(line.slice(6)); } catch { /* skip */ }
try {
yield JSON.parse(line.slice(6));
} catch {
/* skip */
}
}
}
}
if (buffer.trim().startsWith("data: ")) {
try { yield JSON.parse(buffer.trim().slice(6)); } catch { /* skip */ }
try {
yield JSON.parse(buffer.trim().slice(6));
} catch {
/* skip */
}
}
}
// ═══════════════════════════════════════════════════
// Chat Attachments
// ═══════════════════════════════════════════════════
export async function uploadAttachments(token, chatId, files) {
const form = new FormData();
for (const file of files) form.append("files", file);
const res = await fetch(`${BASE}/chats/${chatId}/attachments`, {
method: "POST", headers: authHeader(token), body: form,
method: "POST",
headers: authHeader(token),
body: form,
});
if (!res.ok) {
const err = await res.json().catch(() => ({}));
......@@ -98,7 +126,12 @@ export function getAttachmentUrl(attachmentId) {
export const deleteAttachment = (token, attachmentId) =>
request("DELETE", `/attachments/${attachmentId}`, token);
export const listKnowledgeBases = (token) => request("GET", "/knowledge", token);
// ═══════════════════════════════════════════════════
// Knowledge Bases
// ═══════════════════════════════════════════════════
export const listKnowledgeBases = (token) =>
request("GET", "/knowledge", token);
export const createKnowledgeBase = (token, name, description = "") =>
request("POST", "/knowledge", token, { name, description });
......@@ -106,14 +139,29 @@ export const createKnowledgeBase = (token, name, description = "") =>
export const getKnowledgeBase = (token, kbId) =>
request("GET", `/knowledge/${kbId}`, token);
export const updateKnowledgeBase = (token, kbId, data) =>
request("PUT", `/knowledge/${kbId}`, token, data);
export const deleteKnowledgeBase = (token, kbId) =>
request("DELETE", `/knowledge/${kbId}`, token);
// ═══════════════════════════════════════════════════
// Knowledge Base Documents
// ═══════════════════════════════════════════════════
export const listKnowledgeDocuments = (token, kbId) =>
request("GET", `/knowledge/${kbId}/documents`, token);
export const deleteKnowledgeDocument = (token, kbId, docId) =>
request("DELETE", `/knowledge/${kbId}/documents/${docId}`, token);
export async function uploadDocuments(token, kbId, files) {
const form = new FormData();
for (const file of files) form.append("files", file);
const res = await fetch(`${BASE}/knowledge/${kbId}/upload`, {
method: "POST", headers: authHeader(token), body: form,
method: "POST",
headers: authHeader(token),
body: form,
});
if (!res.ok) {
const err = await res.json().catch(() => ({}));
......@@ -125,16 +173,30 @@ export async function uploadDocuments(token, kbId, files) {
export const uploadDocument = (token, kbId, file) =>
uploadDocuments(token, kbId, [file]);
// ═══════════════════════════════════════════════════
// Admin
// ═══════════════════════════════════════════════════
export const adminStats = (token) => request("GET", "/admin/stats", token);
export const adminListUsers = (token) => request("GET", "/admin/users", token);
export const adminCreateUser = (token, data) => request("POST", "/admin/users", token, data);
export const adminUpdateUser = (token, userId, data) => request("PUT", `/admin/users/${userId}`, token, data);
export const adminDeleteUser = (token, userId) => request("DELETE", `/admin/users/${userId}`, token);
export const adminListChats = (token) => request("GET", "/admin/chats", token);
export const adminListUsers = (token) =>
request("GET", "/admin/users", token);
export const adminCreateUser = (token, data) =>
request("POST", "/admin/users", token, data);
export const adminUpdateUser = (token, userId, data) =>
request("PUT", `/admin/users/${userId}`, token, data);
export const adminDeleteUser = (token, userId) =>
request("DELETE", `/admin/users/${userId}`, token);
export const adminListChats = (token) =>
request("GET", "/admin/chats", token);
// ═══════════════════════════════════════════════════
// Code Download
// ═══════════════════════════════════════════════════
export async function downloadZip(token, markdown) {
const res = await fetch(`${BASE}/files/download-zip`, {
method: "POST", headers: headers(token),
method: "POST",
headers: headers(token),
body: JSON.stringify({ markdown }),
});
if (!res.ok) throw new Error("Download failed");
......
import React, { useEffect, useState } from "react";
import { useApp } from "../store";
import { listChats, createChat, checkGenerating } from "../api";
import * as streamManager from "../streamManager";
import { listChats, createChat } from "../api";
import Sidebar from "../components/Sidebar";
import ChatView from "../components/ChatView";
import { Flame, Menu, Plus, MessageSquare } from "lucide-react";
import { Flame, BookOpen, Shield } from "lucide-react";
import { useNavigate } from "react-router-dom";
export default function ChatPage() {
const { state, dispatch } = useApp();
const navigate = useNavigate();
const [activeChatId, setActiveChatId] = useState(null);
const [sidebarOpen, setSidebarOpen] = useState(false);
useEffect(() => {
(async () => {
try {
const chats = await listChats(state.token);
dispatch({ type: "SET_CHATS", chats });
if (chats.length && !activeChatId) {
const firstId = chats[0].id;
setActiveChatId(firstId);
// Check if background generation is active
try {
const { active } = await checkGenerating(state.token, firstId);
if (active) streamManager.reconnectStream({ token: state.token, chatId: firstId });
} catch { /* ignore */ }
if (chats.length > 0 && !activeChatId) {
setActiveChatId(chats[0].id);
}
} catch { /* ignore */ }
} catch { }
})();
}, [state.token, dispatch]);
......@@ -33,84 +29,72 @@ export default function ChatPage() {
const chat = await createChat(state.token);
dispatch({ type: "ADD_CHAT", chat });
setActiveChatId(chat.id);
dispatch({ type: "SET_SIDEBAR_OPEN", open: false });
} catch { /* ignore */ }
setSidebarOpen(false);
} catch { }
}
async function handleSelectChat(id) {
setActiveChatId(id);
dispatch({ type: "SET_SIDEBAR_OPEN", open: false });
// Check for active background generation on this chat
try {
const { active } = await checkGenerating(state.token, id);
if (active && !streamManager.isStreaming(id)) {
streamManager.reconnectStream({ token: state.token, chatId: id });
}
} catch { /* ignore */ }
function handleSelectChat(chatId) {
setActiveChatId(chatId);
setSidebarOpen(false);
}
return (
<div className="h-dvh flex overflow-hidden bg-anton-bg">
{/* Mobile overlay */}
{state.sidebarOpen && (
<div
className="fixed inset-0 bg-black/60 z-30 lg:hidden"
onClick={() => dispatch({ type: "SET_SIDEBAR_OPEN", open: false })}
/>
)}
<div className="h-dvh flex bg-anton-bg text-anton-text overflow-hidden">
{/* Sidebar */}
<div className={`
fixed inset-y-0 left-0 z-40 w-72 transform transition-transform duration-300 ease-in-out
lg:relative lg:translate-x-0 lg:w-72 lg:shrink-0
${state.sidebarOpen ? "translate-x-0" : "-translate-x-full"}
`}>
<Sidebar
activeChatId={activeChatId}
onSelectChat={handleSelectChat}
onNewChat={handleNewChat}
/>
</div>
<Sidebar
activeChatId={activeChatId}
onSelectChat={handleSelectChat}
onNewChat={handleNewChat}
isOpen={sidebarOpen}
onClose={() => setSidebarOpen(false)}
/>
{/* Main content */}
<div className="flex-1 flex flex-col min-w-0">
{/* Mobile header */}
<div className="flex items-center gap-3 px-4 py-3 border-b border-anton-border bg-anton-surface lg:hidden">
<button
onClick={() => dispatch({ type: "SET_SIDEBAR_OPEN", open: true })}
className="p-1.5 rounded-lg text-anton-muted hover:text-white hover:bg-anton-card transition"
>
<Menu size={20} />
{/* Main */}
<div className="flex-1 flex flex-col min-h-0 min-w-0">
{/* Top bar */}
<div className="border-b border-anton-border bg-anton-surface px-3 py-2 flex items-center gap-2">
<button onClick={() => setSidebarOpen(true)} className="sm:hidden p-1.5 rounded-lg text-anton-muted hover:text-white hover:bg-anton-card transition">
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round"><line x1="3" y1="6" x2="21" y2="6" /><line x1="3" y1="12" x2="21" y2="12" /><line x1="3" y1="18" x2="21" y2="18" /></svg>
</button>
<div className="flex-1 min-w-0">
<h1 className="text-sm font-semibold text-white truncate">
{state.chats.find((c) => c.id === activeChatId)?.title || "Son of Anton"}
</h1>
<div className="w-7 h-7 rounded-lg bg-gradient-to-br from-anton-accent to-red-600 flex items-center justify-center">
<Flame size={14} className="text-white" />
</div>
<span className="text-sm font-semibold text-white truncate flex-1">
{state.chats.find((c) => c.id === activeChatId)?.title || "Son of Anton"}
</span>
<button
onClick={handleNewChat}
className="p-1.5 rounded-lg text-anton-muted hover:text-white hover:bg-anton-card transition"
onClick={() => navigate("/knowledge")}
className="flex items-center gap-1 px-2 py-1 rounded-lg text-xs text-anton-muted hover:text-green-400 hover:bg-green-500/10 transition"
title="Knowledge Bases"
>
<Plus size={20} />
<BookOpen size={14} />
<span className="hidden sm:inline">Knowledge</span>
</button>
{state.user?.role === "superadmin" && (
<button
onClick={() => navigate("/admin")}
className="flex items-center gap-1 px-2 py-1 rounded-lg text-xs text-anton-muted hover:text-anton-accent hover:bg-anton-accent/10 transition"
title="Admin Panel"
>
<Shield size={14} />
<span className="hidden sm:inline">Admin</span>
</button>
)}
</div>
{/* Chat or empty state */}
{/* Chat area */}
{activeChatId ? (
<ChatView chatId={activeChatId} />
) : (
<div className="flex-1 flex items-center justify-center p-8">
<div className="text-center max-w-md">
<div className="w-20 h-20 rounded-2xl bg-gradient-to-br from-anton-accent to-red-600 flex items-center justify-center mx-auto mb-6 shadow-lg shadow-anton-accent/20">
<Flame size={40} className="text-white" />
<div className="flex-1 flex items-center justify-center">
<div className="text-center">
<div className="w-16 h-16 rounded-2xl bg-gradient-to-br from-anton-accent to-red-600 flex items-center justify-center mx-auto mb-4 shadow-lg shadow-anton-accent/20">
<Flame size={32} className="text-white" />
</div>
<h2 className="text-2xl font-bold text-white mb-2">Son of Anton</h2>
<p className="text-anton-muted mb-6">Avatar of All Elements of Code</p>
<button
onClick={handleNewChat}
className="inline-flex items-center gap-2 px-6 py-3 bg-anton-accent text-white rounded-xl hover:opacity-90 transition font-medium"
>
<MessageSquare size={18} /> Start a conversation
<h2 className="text-xl font-bold text-white mb-2">Son of Anton</h2>
<p className="text-anton-muted text-sm mb-6">Avatar of All Elements of Code</p>
<button onClick={handleNewChat} className="px-6 py-2.5 rounded-xl bg-anton-accent text-white font-medium hover:opacity-80 transition">
Start a Chat
</button>
</div>
</div>
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment