File Sync and repoint to WebDAV folder

docker-compose.yml

@@ -9,13 +9,13 @@ services:
     restart: unless-stopped
     environment:
       DATABASE_URL: postgresql+asyncpg://postgres:${DB_PASSWORD}@lifeos-db:5432/lifeos_prod
-      FILE_STORAGE_PATH: /opt/lifeos/files/prod
+      FILE_STORAGE_PATH: /opt/lifeos/webdav
       ENVIRONMENT: production
     command: uvicorn main:app --host 0.0.0.0 --port 8002 --workers 1
     ports:
       - "8002:8002"
     volumes:
-      - /opt/lifeos/prod/files:/opt/lifeos/files/prod
+      - /opt/lifeos/webdav:/opt/lifeos/webdav
     networks:
       - lifeos_network
     depends_on:
@@ -29,13 +29,13 @@ services:
     restart: unless-stopped
     environment:
       DATABASE_URL: postgresql+asyncpg://postgres:${DB_PASSWORD}@lifeos-db:5432/lifeos_dev
-      FILE_STORAGE_PATH: /opt/lifeos/files/dev
+      FILE_STORAGE_PATH: /opt/lifeos/webdav
       ENVIRONMENT: development
     command: uvicorn main:app --host 0.0.0.0 --port 8003 --workers 1 --reload
     ports:
       - "8003:8003"
     volumes:
-      - /opt/lifeos/dev/files:/opt/lifeos/files/dev
+      - /opt/lifeos/webdav:/opt/lifeos/webdav
       - .:/app  # hot reload in dev
     networks:
       - lifeos_network

routers/files.py

@@ -1,12 +1,11 @@
-"""Files: upload, download, list, preview, and polymorphic entity attachment."""
+"""Files: upload, download, list, preview, folder-aware storage, and WebDAV sync."""
 
 import os
-import uuid
-import shutil
+import mimetypes
 from pathlib import Path
 from fastapi import APIRouter, Request, Form, Depends, UploadFile, File as FastAPIFile
 from fastapi.templating import Jinja2Templates
-from fastapi.responses import RedirectResponse, FileResponse
+from fastapi.responses import RedirectResponse, FileResponse, HTMLResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import text
 from typing import Optional
@@ -18,7 +17,7 @@ from core.sidebar import get_sidebar_data
 router = APIRouter(prefix="/files", tags=["files"])
 templates = Jinja2Templates(directory="templates")
 
-FILE_STORAGE_PATH = os.getenv("FILE_STORAGE_PATH", "/opt/lifeos/files/dev")
+FILE_STORAGE_PATH = os.getenv("FILE_STORAGE_PATH", "/opt/lifeos/webdav")
 
 # Ensure storage dir exists
 Path(FILE_STORAGE_PATH).mkdir(parents=True, exist_ok=True)
@@ -29,16 +28,120 @@ PREVIEWABLE = {
     "application/pdf", "text/plain", "text/html", "text/csv",
 }
 
+# Files to skip during sync
+SKIP_FILES = {".DS_Store", "Thumbs.db", ".gitkeep", "desktop.ini"}
+
+
+def _resolve_path(item):
+    """Resolve a DB record's relative storage_path to an absolute path."""
+    return os.path.join(FILE_STORAGE_PATH, item["storage_path"])
+
+
+def get_folders():
+    """Walk FILE_STORAGE_PATH and return sorted list of relative folder paths."""
+    folders = []
+    for dirpath, dirnames, _filenames in os.walk(FILE_STORAGE_PATH):
+        # Skip hidden directories
+        dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+        rel = os.path.relpath(dirpath, FILE_STORAGE_PATH)
+        if rel != ".":
+            folders.append(rel)
+    return sorted(folders)
+
+
+def resolve_collision(folder_abs, filename):
+    """If filename exists in folder_abs, return name (2).ext, name (3).ext, etc."""
+    target = os.path.join(folder_abs, filename)
+    if not os.path.exists(target):
+        return filename
+
+    name, ext = os.path.splitext(filename)
+    counter = 2
+    while True:
+        candidate = f"{name} ({counter}){ext}"
+        if not os.path.exists(os.path.join(folder_abs, candidate)):
+            return candidate
+        counter += 1
+
+
+async def sync_files(db: AsyncSession):
+    """Sync filesystem state with the database.
+
+    - Files on disk not in DB → create record
+    - Active DB records with missing files → soft-delete
+    Returns dict with added/removed counts.
+    """
+    added = 0
+    removed = 0
+
+    # Build set of all relative file paths on disk
+    disk_files = set()
+    for dirpath, dirnames, filenames in os.walk(FILE_STORAGE_PATH):
+        dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+        for fname in filenames:
+            if fname in SKIP_FILES or fname.startswith("."):
+                continue
+            abs_path = os.path.join(dirpath, fname)
+            rel_path = os.path.relpath(abs_path, FILE_STORAGE_PATH)
+            disk_files.add(rel_path)
+
+    # Get ALL DB records (including soft-deleted) to avoid re-creating deleted files
+    result = await db.execute(text(
+        "SELECT id, storage_path, is_deleted FROM files"
+    ))
+    db_records = [dict(r._mapping) for r in result]
+
+    # Build lookup sets
+    all_db_paths = {r["storage_path"] for r in db_records}
+    active_db_paths = {r["storage_path"] for r in db_records if not r["is_deleted"]}
+
+    # New on disk, not in DB at all → create record
+    new_files = disk_files - all_db_paths
+    for rel_path in new_files:
+        abs_path = os.path.join(FILE_STORAGE_PATH, rel_path)
+        filename = os.path.basename(rel_path)
+        mime_type = mimetypes.guess_type(filename)[0]
+        try:
+            size_bytes = os.path.getsize(abs_path)
+        except OSError:
+            continue
+
+        repo = BaseRepository("files", db)
+        await repo.create({
+            "filename": filename,
+            "original_filename": filename,
+            "storage_path": rel_path,
+            "mime_type": mime_type,
+            "size_bytes": size_bytes,
+        })
+        added += 1
+
+    # Active in DB but missing from disk → soft-delete
+    missing_files = active_db_paths - disk_files
+    for record in db_records:
+        if record["storage_path"] in missing_files and not record["is_deleted"]:
+            repo = BaseRepository("files", db)
+            await repo.soft_delete(record["id"])
+            removed += 1
+
+    return {"added": added, "removed": removed}
+
 
 @router.get("/")
 async def list_files(
     request: Request,
+    folder: Optional[str] = None,
     context_type: Optional[str] = None,
     context_id: Optional[str] = None,
     db: AsyncSession = Depends(get_db),
 ):
     sidebar = await get_sidebar_data(db)
 
+    # Auto-sync on page load
+    sync_result = await sync_files(db)
+
+    folders = get_folders()
+
     if context_type and context_id:
         # Files attached to a specific entity
         result = await db.execute(text("""
@@ -48,19 +151,40 @@ async def list_files(
             WHERE f.is_deleted = false AND fm.context_type = :ct AND fm.context_id = :cid
             ORDER BY f.created_at DESC
         """), {"ct": context_type, "cid": context_id})
+    elif folder is not None:
+        if folder == "":
+            # Root folder: files with no directory separator in storage_path
+            result = await db.execute(text("""
+                SELECT * FROM files
+                WHERE is_deleted = false AND storage_path NOT LIKE '%/%'
+                ORDER BY created_at DESC
+            """))
+        else:
+            # Specific folder: storage_path starts with folder/
+            result = await db.execute(text("""
+                SELECT * FROM files
+                WHERE is_deleted = false AND storage_path LIKE :prefix
+                ORDER BY created_at DESC
+            """), {"prefix": folder + "/%"})
     else:
         # All files
         result = await db.execute(text("""
-            SELECT f.* FROM files f
-            WHERE f.is_deleted = false
-            ORDER BY f.created_at DESC
-            LIMIT 100
+            SELECT * FROM files
+            WHERE is_deleted = false
+            ORDER BY created_at DESC
         """))
 
     items = [dict(r._mapping) for r in result]
 
+    # Add derived folder field for display
+    for item in items:
+        dirname = os.path.dirname(item["storage_path"])
+        item["folder"] = dirname if dirname else "/"
+
     return templates.TemplateResponse("files.html", {
         "request": request, "sidebar": sidebar, "items": items,
+        "folders": folders, "current_folder": folder,
+        "sync_result": sync_result,
         "context_type": context_type or "",
         "context_id": context_id or "",
         "page_title": "Files", "active_nav": "files",
@@ -70,13 +194,16 @@ async def list_files(
 @router.get("/upload")
 async def upload_form(
     request: Request,
+    folder: Optional[str] = None,
     context_type: Optional[str] = None,
     context_id: Optional[str] = None,
     db: AsyncSession = Depends(get_db),
 ):
     sidebar = await get_sidebar_data(db)
+    folders = get_folders()
     return templates.TemplateResponse("file_upload.html", {
         "request": request, "sidebar": sidebar,
+        "folders": folders, "prefill_folder": folder or "",
         "context_type": context_type or "",
         "context_id": context_id or "",
         "page_title": "Upload File", "active_nav": "files",
@@ -89,19 +216,41 @@ async def upload_file(
     file: UploadFile = FastAPIFile(...),
     description: Optional[str] = Form(None),
     tags: Optional[str] = Form(None),
+    folder: Optional[str] = Form(None),
+    new_folder: Optional[str] = Form(None),
     context_type: Optional[str] = Form(None),
     context_id: Optional[str] = Form(None),
     db: AsyncSession = Depends(get_db),
 ):
-    # Generate storage filename
-    file_uuid = str(uuid.uuid4())
+    # Determine target folder
+    target_folder = ""
+    if new_folder and new_folder.strip():
+        target_folder = new_folder.strip().strip("/")
+    elif folder and folder.strip():
+        target_folder = folder.strip()
+
+    # Build absolute folder path and ensure it exists
+    if target_folder:
+        folder_abs = os.path.join(FILE_STORAGE_PATH, target_folder)
+    else:
+        folder_abs = FILE_STORAGE_PATH
+    os.makedirs(folder_abs, exist_ok=True)
+
+    # Use original filename, handle collisions
     original = file.filename or "unknown"
     safe_name = original.replace("/", "_").replace("\\", "_")
-    storage_name = f"{file_uuid}_{safe_name}"
-    storage_path = os.path.join(FILE_STORAGE_PATH, storage_name)
+    final_name = resolve_collision(folder_abs, safe_name)
+
+    # Build relative storage path
+    if target_folder:
+        storage_path = os.path.join(target_folder, final_name)
+    else:
+        storage_path = final_name
+
+    abs_path = os.path.join(FILE_STORAGE_PATH, storage_path)
 
     # Save to disk
-    with open(storage_path, "wb") as f:
+    with open(abs_path, "wb") as f:
         content = await file.read()
         f.write(content)
 
@@ -110,7 +259,7 @@ async def upload_file(
     # Insert file record
     repo = BaseRepository("files", db)
     data = {
-        "filename": storage_name,
+        "filename": final_name,
         "original_filename": original,
         "storage_path": storage_path,
         "mime_type": file.content_type,
@@ -139,15 +288,26 @@ async def upload_file(
     return RedirectResponse(url="/files", status_code=303)
 
 
+@router.post("/sync")
+async def manual_sync(request: Request, db: AsyncSession = Depends(get_db)):
+    """Manual sync trigger."""
+    await sync_files(db)
+    return RedirectResponse(url="/files", status_code=303)
+
+
 @router.get("/{file_id}/download")
 async def download_file(file_id: str, db: AsyncSession = Depends(get_db)):
     repo = BaseRepository("files", db)
     item = await repo.get(file_id)
-    if not item or not os.path.exists(item["storage_path"]):
+    if not item:
+        return RedirectResponse(url="/files", status_code=303)
+
+    abs_path = _resolve_path(item)
+    if not os.path.exists(abs_path):
         return RedirectResponse(url="/files", status_code=303)
 
     return FileResponse(
-        path=item["storage_path"],
+        path=abs_path,
         filename=item["original_filename"],
         media_type=item.get("mime_type") or "application/octet-stream",
     )
@@ -163,10 +323,12 @@ async def preview_file(file_id: str, request: Request, db: AsyncSession = Depend
         return RedirectResponse(url="/files", status_code=303)
 
     can_preview = item.get("mime_type", "") in PREVIEWABLE
+    folder = os.path.dirname(item["storage_path"])
 
     return templates.TemplateResponse("file_preview.html", {
         "request": request, "sidebar": sidebar, "item": item,
         "can_preview": can_preview,
+        "folder": folder if folder else "/",
         "page_title": item["original_filename"], "active_nav": "files",
     })
 
@@ -176,13 +338,34 @@ async def serve_file(file_id: str, db: AsyncSession = Depends(get_db)):
     """Serve file inline (for img src, iframe, etc)."""
     repo = BaseRepository("files", db)
     item = await repo.get(file_id)
-    if not item or not os.path.exists(item["storage_path"]):
+    if not item:
         return RedirectResponse(url="/files", status_code=303)
 
-    return FileResponse(
-        path=item["storage_path"],
-        media_type=item.get("mime_type") or "application/octet-stream",
-    )
+    abs_path = _resolve_path(item)
+    if not os.path.exists(abs_path):
+        return RedirectResponse(url="/files", status_code=303)
+
+    mime = item.get("mime_type") or "application/octet-stream"
+
+    # Wrap text files in HTML with forced white background / dark text
+    if mime.startswith("text/"):
+        try:
+            with open(abs_path, "r", errors="replace") as f:
+                text_content = f.read()
+        except Exception:
+            return FileResponse(path=abs_path, media_type=mime)
+
+        from html import escape
+        html = (
+            '<!DOCTYPE html><html><head><meta charset="utf-8">'
+            '<style>body{background:#fff;color:#1a1a1a;font-family:monospace;'
+            'font-size:14px;padding:16px;margin:0;white-space:pre-wrap;'
+            'word-wrap:break-word;}</style></head><body>'
+            f'{escape(text_content)}</body></html>'
+        )
+        return HTMLResponse(content=html)
+
+    return FileResponse(path=abs_path, media_type=mime)
 
 
 @router.post("/{file_id}/delete")

templates/file_preview.html

@@ -3,6 +3,10 @@
 <div class="breadcrumb">
     <a href="/files">Files</a>
     <span class="sep">/</span>
+    {% if folder and folder != '/' %}
+    <a href="/files?folder={{ folder }}">{{ folder }}</a>
+    <span class="sep">/</span>
+    {% endif %}
     <span>{{ item.original_filename }}</span>
 </div>
 
@@ -17,6 +21,7 @@
 </div>
 
 <div class="detail-meta mt-2">
+    <span class="detail-meta-item">Folder: {{ folder }}</span>
     {% if item.mime_type %}<span class="detail-meta-item">Type: {{ item.mime_type }}</span>{% endif %}
     {% if item.size_bytes %}<span class="detail-meta-item">Size: {{ "%.1f"|format(item.size_bytes / 1024) }} KB</span>{% endif %}
     {% if item.description %}<span class="detail-meta-item">{{ item.description }}</span>{% endif %}
@@ -34,7 +39,7 @@
     {% elif item.mime_type == 'application/pdf' %}
     <iframe src="/files/{{ item.id }}/serve" style="width: 100%; height: 600px; border: none; border-radius: var(--radius);"></iframe>
     {% elif item.mime_type and item.mime_type.startswith('text/') %}
-    <iframe src="/files/{{ item.id }}/serve" style="width: 100%; height: 400px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface2);"></iframe>
+    <iframe src="/files/{{ item.id }}/serve" style="width: 100%; height: 400px; border: 1px solid var(--border); border-radius: var(--radius); background: #fff;"></iframe>
     {% endif %}
 </div>
 {% else %}

templates/file_upload.html

@@ -17,6 +17,21 @@
                 <input type="file" name="file" class="form-input" required>
             </div>
 
+            <div class="form-group">
+                <label class="form-label">Folder</label>
+                <select name="folder" class="form-input">
+                    <option value="">/ (root)</option>
+                    {% for f in folders %}
+                    <option value="{{ f }}" {{ 'selected' if prefill_folder == f }}>{{ f }}</option>
+                    {% endfor %}
+                </select>
+            </div>
+
+            <div class="form-group">
+                <label class="form-label">New Folder</label>
+                <input type="text" name="new_folder" class="form-input" placeholder="Or create new folder...">
+            </div>
+
             <div class="form-group full-width">
                 <label class="form-label">Description</label>
                 <input type="text" name="description" class="form-input" placeholder="Optional description...">

templates/files.html

@@ -2,9 +2,30 @@
 {% block content %}
 <div class="page-header">
     <h1 class="page-title">Files<span class="page-count">{{ items|length }}</span></h1>
-    <a href="/files/upload{{ '?context_type=' ~ context_type ~ '&context_id=' ~ context_id if context_type }}" class="btn btn-primary">+ Upload File</a>
+    <div class="flex gap-2">
+        <form action="/files/sync" method="post" style="display:inline">
+            <button type="submit" class="btn btn-secondary">Sync Files</button>
+        </form>
+        <a href="/files/upload{{ '?folder=' ~ current_folder if current_folder }}{{ '?context_type=' ~ context_type ~ '&context_id=' ~ context_id if context_type }}" class="btn btn-primary">+ Upload File</a>
+    </div>
 </div>
 
+{% if sync_result and (sync_result.added > 0 or sync_result.removed > 0) %}
+<div class="flash-message" style="background: var(--accent-soft); border: 1px solid var(--accent); border-radius: var(--radius); padding: 8px 12px; margin-bottom: 16px; color: var(--text); font-size: 0.85rem;">
+    Synced: {{ sync_result.added }} file{{ 's' if sync_result.added != 1 }} added, {{ sync_result.removed }} removed
+</div>
+{% endif %}
+
+{% if folders %}
+<div class="filter-bar" style="display: flex; gap: 6px; flex-wrap: wrap; margin-bottom: 16px;">
+    <a href="/files" class="btn btn-xs {{ 'btn-primary' if current_folder is none else 'btn-ghost' }}">All</a>
+    <a href="/files?folder=" class="btn btn-xs {{ 'btn-primary' if current_folder is not none and current_folder == '' else 'btn-ghost' }}">/</a>
+    {% for f in folders %}
+    <a href="/files?folder={{ f }}" class="btn btn-xs {{ 'btn-primary' if current_folder == f else 'btn-ghost' }}">{{ f }}</a>
+    {% endfor %}
+</div>
+{% endif %}
+
 {% if items %}
 <div class="card">
     {% for item in items %}
@@ -12,6 +33,7 @@
         <span class="row-title">
             <a href="/files/{{ item.id }}/preview">{{ item.original_filename }}</a>
         </span>
+        <span class="row-meta" style="color: var(--muted); font-size: 0.8rem;">{{ item.folder }}</span>
         {% if item.mime_type %}
         <span class="row-tag">{{ item.mime_type.split('/')|last }}</span>
         {% endif %}
@@ -33,7 +55,7 @@
 {% else %}
 <div class="empty-state">
     <div class="empty-state-icon">&#128193;</div>
-    <div class="empty-state-text">No files uploaded yet</div>
+    <div class="empty-state-text">No files{{ ' in this folder' if current_folder is not none else ' uploaded yet' }}</div>
     <a href="/files/upload" class="btn btn-primary">Upload First File</a>
 </div>
 {% endif %}