File Sync and repoint to WebDAV folder

routers/files.py

@@ -1,12 +1,11 @@
-"""Files: upload, download, list, preview, and polymorphic entity attachment."""
+"""Files: upload, download, list, preview, folder-aware storage, and WebDAV sync."""
 
 import os
-import uuid
-import shutil
+import mimetypes
 from pathlib import Path
 from fastapi import APIRouter, Request, Form, Depends, UploadFile, File as FastAPIFile
 from fastapi.templating import Jinja2Templates
-from fastapi.responses import RedirectResponse, FileResponse
+from fastapi.responses import RedirectResponse, FileResponse, HTMLResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import text
 from typing import Optional
@@ -18,7 +17,7 @@ from core.sidebar import get_sidebar_data
 router = APIRouter(prefix="/files", tags=["files"])
 templates = Jinja2Templates(directory="templates")
 
-FILE_STORAGE_PATH = os.getenv("FILE_STORAGE_PATH", "/opt/lifeos/files/dev")
+FILE_STORAGE_PATH = os.getenv("FILE_STORAGE_PATH", "/opt/lifeos/webdav")
 
 # Ensure storage dir exists
 Path(FILE_STORAGE_PATH).mkdir(parents=True, exist_ok=True)
@@ -29,16 +28,120 @@ PREVIEWABLE = {
     "application/pdf", "text/plain", "text/html", "text/csv",
 }
 
+# Files to skip during sync
+SKIP_FILES = {".DS_Store", "Thumbs.db", ".gitkeep", "desktop.ini"}
+
+
+def _resolve_path(item):
+    """Resolve a DB record's relative storage_path to an absolute path."""
+    return os.path.join(FILE_STORAGE_PATH, item["storage_path"])
+
+
+def get_folders():
+    """Walk FILE_STORAGE_PATH and return sorted list of relative folder paths."""
+    folders = []
+    for dirpath, dirnames, _filenames in os.walk(FILE_STORAGE_PATH):
+        # Skip hidden directories
+        dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+        rel = os.path.relpath(dirpath, FILE_STORAGE_PATH)
+        if rel != ".":
+            folders.append(rel)
+    return sorted(folders)
+
+
+def resolve_collision(folder_abs, filename):
+    """If filename exists in folder_abs, return name (2).ext, name (3).ext, etc."""
+    target = os.path.join(folder_abs, filename)
+    if not os.path.exists(target):
+        return filename
+
+    name, ext = os.path.splitext(filename)
+    counter = 2
+    while True:
+        candidate = f"{name} ({counter}){ext}"
+        if not os.path.exists(os.path.join(folder_abs, candidate)):
+            return candidate
+        counter += 1
+
+
+async def sync_files(db: AsyncSession):
+    """Sync filesystem state with the database.
+
+    - Files on disk not in DB → create record
+    - Active DB records with missing files → soft-delete
+    Returns dict with added/removed counts.
+    """
+    added = 0
+    removed = 0
+
+    # Build set of all relative file paths on disk
+    disk_files = set()
+    for dirpath, dirnames, filenames in os.walk(FILE_STORAGE_PATH):
+        dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+        for fname in filenames:
+            if fname in SKIP_FILES or fname.startswith("."):
+                continue
+            abs_path = os.path.join(dirpath, fname)
+            rel_path = os.path.relpath(abs_path, FILE_STORAGE_PATH)
+            disk_files.add(rel_path)
+
+    # Get ALL DB records (including soft-deleted) to avoid re-creating deleted files
+    result = await db.execute(text(
+        "SELECT id, storage_path, is_deleted FROM files"
+    ))
+    db_records = [dict(r._mapping) for r in result]
+
+    # Build lookup sets
+    all_db_paths = {r["storage_path"] for r in db_records}
+    active_db_paths = {r["storage_path"] for r in db_records if not r["is_deleted"]}
+
+    # New on disk, not in DB at all → create record
+    new_files = disk_files - all_db_paths
+    for rel_path in new_files:
+        abs_path = os.path.join(FILE_STORAGE_PATH, rel_path)
+        filename = os.path.basename(rel_path)
+        mime_type = mimetypes.guess_type(filename)[0]
+        try:
+            size_bytes = os.path.getsize(abs_path)
+        except OSError:
+            continue
+
+        repo = BaseRepository("files", db)
+        await repo.create({
+            "filename": filename,
+            "original_filename": filename,
+            "storage_path": rel_path,
+            "mime_type": mime_type,
+            "size_bytes": size_bytes,
+        })
+        added += 1
+
+    # Active in DB but missing from disk → soft-delete
+    missing_files = active_db_paths - disk_files
+    for record in db_records:
+        if record["storage_path"] in missing_files and not record["is_deleted"]:
+            repo = BaseRepository("files", db)
+            await repo.soft_delete(record["id"])
+            removed += 1
+
+    return {"added": added, "removed": removed}
+
 
 @router.get("/")
 async def list_files(
     request: Request,
+    folder: Optional[str] = None,
     context_type: Optional[str] = None,
     context_id: Optional[str] = None,
     db: AsyncSession = Depends(get_db),
 ):
     sidebar = await get_sidebar_data(db)
 
+    # Auto-sync on page load
+    sync_result = await sync_files(db)
+
+    folders = get_folders()
+
     if context_type and context_id:
         # Files attached to a specific entity
         result = await db.execute(text("""
@@ -48,19 +151,40 @@ async def list_files(
             WHERE f.is_deleted = false AND fm.context_type = :ct AND fm.context_id = :cid
             ORDER BY f.created_at DESC
         """), {"ct": context_type, "cid": context_id})
+    elif folder is not None:
+        if folder == "":
+            # Root folder: files with no directory separator in storage_path
+            result = await db.execute(text("""
+                SELECT * FROM files
+                WHERE is_deleted = false AND storage_path NOT LIKE '%/%'
+                ORDER BY created_at DESC
+            """))
+        else:
+            # Specific folder: storage_path starts with folder/
+            result = await db.execute(text("""
+                SELECT * FROM files
+                WHERE is_deleted = false AND storage_path LIKE :prefix
+                ORDER BY created_at DESC
+            """), {"prefix": folder + "/%"})
     else:
         # All files
         result = await db.execute(text("""
-            SELECT f.* FROM files f
-            WHERE f.is_deleted = false
-            ORDER BY f.created_at DESC
-            LIMIT 100
+            SELECT * FROM files
+            WHERE is_deleted = false
+            ORDER BY created_at DESC
         """))
 
     items = [dict(r._mapping) for r in result]
 
+    # Add derived folder field for display
+    for item in items:
+        dirname = os.path.dirname(item["storage_path"])
+        item["folder"] = dirname if dirname else "/"
+
     return templates.TemplateResponse("files.html", {
         "request": request, "sidebar": sidebar, "items": items,
+        "folders": folders, "current_folder": folder,
+        "sync_result": sync_result,
         "context_type": context_type or "",
         "context_id": context_id or "",
         "page_title": "Files", "active_nav": "files",
@@ -70,13 +194,16 @@ async def list_files(
 @router.get("/upload")
 async def upload_form(
     request: Request,
+    folder: Optional[str] = None,
     context_type: Optional[str] = None,
     context_id: Optional[str] = None,
     db: AsyncSession = Depends(get_db),
 ):
     sidebar = await get_sidebar_data(db)
+    folders = get_folders()
     return templates.TemplateResponse("file_upload.html", {
         "request": request, "sidebar": sidebar,
+        "folders": folders, "prefill_folder": folder or "",
         "context_type": context_type or "",
         "context_id": context_id or "",
         "page_title": "Upload File", "active_nav": "files",
@@ -89,19 +216,41 @@ async def upload_file(
     file: UploadFile = FastAPIFile(...),
     description: Optional[str] = Form(None),
     tags: Optional[str] = Form(None),
+    folder: Optional[str] = Form(None),
+    new_folder: Optional[str] = Form(None),
     context_type: Optional[str] = Form(None),
     context_id: Optional[str] = Form(None),
     db: AsyncSession = Depends(get_db),
 ):
-    # Generate storage filename
-    file_uuid = str(uuid.uuid4())
+    # Determine target folder
+    target_folder = ""
+    if new_folder and new_folder.strip():
+        target_folder = new_folder.strip().strip("/")
+    elif folder and folder.strip():
+        target_folder = folder.strip()
+
+    # Build absolute folder path and ensure it exists
+    if target_folder:
+        folder_abs = os.path.join(FILE_STORAGE_PATH, target_folder)
+    else:
+        folder_abs = FILE_STORAGE_PATH
+    os.makedirs(folder_abs, exist_ok=True)
+
+    # Use original filename, handle collisions
     original = file.filename or "unknown"
     safe_name = original.replace("/", "_").replace("\\", "_")
-    storage_name = f"{file_uuid}_{safe_name}"
-    storage_path = os.path.join(FILE_STORAGE_PATH, storage_name)
+    final_name = resolve_collision(folder_abs, safe_name)
+
+    # Build relative storage path
+    if target_folder:
+        storage_path = os.path.join(target_folder, final_name)
+    else:
+        storage_path = final_name
+
+    abs_path = os.path.join(FILE_STORAGE_PATH, storage_path)
 
     # Save to disk
-    with open(storage_path, "wb") as f:
+    with open(abs_path, "wb") as f:
         content = await file.read()
         f.write(content)
 
@@ -110,7 +259,7 @@ async def upload_file(
     # Insert file record
     repo = BaseRepository("files", db)
     data = {
-        "filename": storage_name,
+        "filename": final_name,
         "original_filename": original,
         "storage_path": storage_path,
         "mime_type": file.content_type,
@@ -139,15 +288,26 @@ async def upload_file(
     return RedirectResponse(url="/files", status_code=303)
 
 
+@router.post("/sync")
+async def manual_sync(request: Request, db: AsyncSession = Depends(get_db)):
+    """Manual sync trigger."""
+    await sync_files(db)
+    return RedirectResponse(url="/files", status_code=303)
+
+
 @router.get("/{file_id}/download")
 async def download_file(file_id: str, db: AsyncSession = Depends(get_db)):
     repo = BaseRepository("files", db)
     item = await repo.get(file_id)
-    if not item or not os.path.exists(item["storage_path"]):
+    if not item:
+        return RedirectResponse(url="/files", status_code=303)
+
+    abs_path = _resolve_path(item)
+    if not os.path.exists(abs_path):
         return RedirectResponse(url="/files", status_code=303)
 
     return FileResponse(
-        path=item["storage_path"],
+        path=abs_path,
         filename=item["original_filename"],
         media_type=item.get("mime_type") or "application/octet-stream",
     )
@@ -163,10 +323,12 @@ async def preview_file(file_id: str, request: Request, db: AsyncSession = Depend
         return RedirectResponse(url="/files", status_code=303)
 
     can_preview = item.get("mime_type", "") in PREVIEWABLE
+    folder = os.path.dirname(item["storage_path"])
 
     return templates.TemplateResponse("file_preview.html", {
         "request": request, "sidebar": sidebar, "item": item,
         "can_preview": can_preview,
+        "folder": folder if folder else "/",
         "page_title": item["original_filename"], "active_nav": "files",
     })
 
@@ -176,13 +338,34 @@ async def serve_file(file_id: str, db: AsyncSession = Depends(get_db)):
     """Serve file inline (for img src, iframe, etc)."""
     repo = BaseRepository("files", db)
     item = await repo.get(file_id)
-    if not item or not os.path.exists(item["storage_path"]):
+    if not item:
         return RedirectResponse(url="/files", status_code=303)
 
-    return FileResponse(
-        path=item["storage_path"],
-        media_type=item.get("mime_type") or "application/octet-stream",
-    )
+    abs_path = _resolve_path(item)
+    if not os.path.exists(abs_path):
+        return RedirectResponse(url="/files", status_code=303)
+
+    mime = item.get("mime_type") or "application/octet-stream"
+
+    # Wrap text files in HTML with forced white background / dark text
+    if mime.startswith("text/"):
+        try:
+            with open(abs_path, "r", errors="replace") as f:
+                text_content = f.read()
+        except Exception:
+            return FileResponse(path=abs_path, media_type=mime)
+
+        from html import escape
+        html = (
+            '<!DOCTYPE html><html><head><meta charset="utf-8">'
+            '<style>body{background:#fff;color:#1a1a1a;font-family:monospace;'
+            'font-size:14px;padding:16px;margin:0;white-space:pre-wrap;'
+            'word-wrap:break-word;}</style></head><body>'
+            f'{escape(text_content)}</body></html>'
+        )
+        return HTMLResponse(content=html)
+
+    return FileResponse(path=abs_path, media_type=mime)
 
 
 @router.post("/{file_id}/delete")