refactor: remove storage, backup, plugin/marketplace features
- Delete app/storage/ (blob_store, vector_store, encryption) - Delete app/marketplace/ (plugin_registry, plugin_review, revenue_share) - Delete routes: backup.py, plugins.py, storage.py, vectors.py - Relocate embed endpoint to POST /chat/embed - Rewrite migration 001 (remove storage/plugin tables) - Delete migration 002 (seed_plugins) - Remove S3/Pinecone/Qdrant env vars from settings - Remove storage/backup quotas from tier_manager - Remove MinIO and Qdrant from docker-compose - Delete tests: test_backup, test_plugins, test_storage - Update README.md and clean .env.example
This commit is contained in:
Roberto Musso
@@ -8,8 +8,7 @@ that could reveal server-side prompt IP:
|
||||
- Internal reasoning markers (<thinking>, <reasoning>, [INST], …)
|
||||
- Exact-match known prompt fingerprints
|
||||
|
||||
Binary responses (storage blobs, backup data) are never touched — the
|
||||
middleware only activates for paths under /api/v1/chat.
|
||||
The middleware only activates for paths under /api/v1/chat.
|
||||
|
||||
Any sanitisation event is logged as a WARNING with the request path and the
|
||||
names of the fields that were modified.
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
"""Backup routes: upload, download, history, and delete E2E-encrypted backups.
|
||||
|
||||
Blobs are stored in S3 via BlobStore. Backup metadata is persisted in the
|
||||
PostgreSQL ``backup_metadata`` table.
|
||||
|
||||
IMPORTANT: GET /history must be declared BEFORE GET / to avoid FastAPI
|
||||
treating "history" as a ``{backup_id}`` path parameter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_current_user
|
||||
from app.billing.tier_manager import tier_manager
|
||||
from app.db import get_session
|
||||
from app.models import BackupMetadata as BackupMetadataModel
|
||||
from app.schemas import BackupMetadata, UserProfile
|
||||
from app.storage.blob_store import BlobStore
|
||||
from app.storage.encryption import reject_if_tampered
|
||||
|
||||
router = APIRouter(prefix="/backup", tags=["backup"])
|
||||
|
||||
_blob_store = BlobStore()
|
||||
|
||||
|
||||
async def _current_backup_bytes(user_id: str, db: AsyncSession) -> int:
|
||||
"""Return total backup bytes stored by *user_id*."""
|
||||
result = await db.execute(
|
||||
select(func.coalesce(func.sum(BackupMetadataModel.size_bytes), 0)).where(
|
||||
BackupMetadataModel.user_id == user_id
|
||||
)
|
||||
)
|
||||
return int(result.scalar_one())
|
||||
|
||||
|
||||
async def _check_backup_quota(
|
||||
user: UserProfile, size_bytes: int, db: AsyncSession
|
||||
) -> None:
|
||||
"""Raise HTTP 402 if the upload would exceed the tier's backup limit."""
|
||||
current = await _current_backup_bytes(user.id, db)
|
||||
tier_manager.enforce_backup_quota(
|
||||
user.tier, current_bytes=current, additional_bytes=size_bytes
|
||||
)
|
||||
|
||||
|
||||
@router.put("")
|
||||
async def upload_backup(
|
||||
request: Request,
|
||||
x_backup_version: int = Header(..., alias="X-Backup-Version"),
|
||||
x_backup_timestamp: int = Header(..., alias="X-Backup-Timestamp"),
|
||||
x_backup_checksum: str = Header(..., alias="X-Backup-Checksum"),
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> dict[str, bool]:
|
||||
"""Upload an E2E-encrypted backup blob.
|
||||
|
||||
Metadata is passed via custom headers; the raw body is the encrypted blob.
|
||||
"""
|
||||
blob = await request.body()
|
||||
reject_if_tampered(blob, x_backup_checksum)
|
||||
await _check_backup_quota(current_user, len(blob), db)
|
||||
|
||||
s3_key = await _blob_store.upload(
|
||||
current_user.id, "backup", str(x_backup_timestamp), blob, x_backup_checksum
|
||||
)
|
||||
|
||||
row = BackupMetadataModel(
|
||||
id=str(uuid.uuid4()),
|
||||
user_id=current_user.id,
|
||||
s3_key=s3_key,
|
||||
version=x_backup_version,
|
||||
timestamp=x_backup_timestamp,
|
||||
checksum=x_backup_checksum,
|
||||
size_bytes=len(blob),
|
||||
)
|
||||
db.add(row)
|
||||
await db.commit()
|
||||
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/history", response_model=list[BackupMetadata])
|
||||
async def backup_history(
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> list[BackupMetadata]:
|
||||
"""Return backup metadata records for the authenticated user (no blob bytes)."""
|
||||
result = await db.execute(
|
||||
select(BackupMetadataModel)
|
||||
.where(BackupMetadataModel.user_id == current_user.id)
|
||||
.order_by(BackupMetadataModel.timestamp.desc())
|
||||
)
|
||||
rows = result.scalars().all()
|
||||
return [
|
||||
BackupMetadata(
|
||||
version=r.version,
|
||||
timestamp=r.timestamp,
|
||||
checksum=r.checksum,
|
||||
chunk_count=1,
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def download_backup(
|
||||
request: Request,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> Response:
|
||||
"""Download the latest backup blob. Supports ``If-Modified-Since``."""
|
||||
result = await db.execute(
|
||||
select(BackupMetadataModel)
|
||||
.where(BackupMetadataModel.user_id == current_user.id)
|
||||
.order_by(BackupMetadataModel.timestamp.desc())
|
||||
.limit(1)
|
||||
)
|
||||
latest = result.scalar_one_or_none()
|
||||
if latest is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No backup found")
|
||||
|
||||
ims_header = request.headers.get("If-Modified-Since")
|
||||
if ims_header:
|
||||
try:
|
||||
ims_dt = parsedate_to_datetime(ims_header)
|
||||
ims_ms = int(ims_dt.timestamp() * 1000)
|
||||
if latest.timestamp <= ims_ms:
|
||||
return Response(status_code=status.HTTP_304_NOT_MODIFIED)
|
||||
except Exception:
|
||||
pass # malformed header — ignore and serve the blob
|
||||
|
||||
blob = await _blob_store.download(current_user.id, latest.s3_key)
|
||||
return Response(
|
||||
content=blob,
|
||||
media_type="application/octet-stream",
|
||||
headers={
|
||||
"X-Backup-Version": str(latest.version),
|
||||
"X-Backup-Timestamp": str(latest.timestamp),
|
||||
"X-Checksum": latest.checksum,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{backup_id}", response_model=dict)
|
||||
async def delete_backup(
|
||||
backup_id: str,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> dict[str, bool]:
|
||||
"""Delete a specific backup by ID."""
|
||||
result = await db.execute(
|
||||
select(BackupMetadataModel).where(
|
||||
BackupMetadataModel.id == backup_id,
|
||||
BackupMetadataModel.user_id == current_user.id,
|
||||
)
|
||||
)
|
||||
target = result.scalar_one_or_none()
|
||||
if target is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Backup not found")
|
||||
|
||||
await _blob_store.delete(current_user.id, target.s3_key)
|
||||
await db.delete(target)
|
||||
await db.commit()
|
||||
|
||||
return {"ok": True}
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Chat routes: POST /chat (REST fallback).
|
||||
"""Chat routes: POST /chat (REST fallback) and POST /chat/embed (text → vector).
|
||||
|
||||
WebSocket chat is handled by the unified device WS endpoint (/api/v1/ws/device).
|
||||
"""
|
||||
@@ -7,14 +7,30 @@ from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.api.deps import get_current_user
|
||||
from app.core.deep_agent import run_home
|
||||
from app.core.llm import embed
|
||||
from app.schemas import ChatRequest, UserProfile
|
||||
|
||||
router = APIRouter(prefix="/chat", tags=["chat"])
|
||||
|
||||
|
||||
# ── Embed helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _EmbedRequest(BaseModel):
|
||||
text: str
|
||||
|
||||
|
||||
class _EmbedResponse(BaseModel):
|
||||
vector: list[float]
|
||||
|
||||
|
||||
# ── Endpoints ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def chat(
|
||||
body: ChatRequest,
|
||||
@@ -27,3 +43,17 @@ async def chat(
|
||||
context=body.context.model_dump(),
|
||||
)
|
||||
return JSONResponse(content={"response": response})
|
||||
|
||||
|
||||
@router.post("/embed", response_model=_EmbedResponse)
|
||||
async def embed_text(
|
||||
body: _EmbedRequest,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
) -> _EmbedResponse:
|
||||
"""Generate a 1536-dim embedding vector for the given text.
|
||||
|
||||
Uses ``text-embedding-3-small`` via OpenAI. Auth required (JWT).
|
||||
Used by Electron (vectordb.ts) for local note search.
|
||||
"""
|
||||
vector = await embed(body.text)
|
||||
return _EmbedResponse(vector=vector)
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
"""Plugins routes: browse and install plugins from the marketplace.
|
||||
|
||||
Backed by ``PluginRegistry`` and ``RevenueShare`` service classes that
|
||||
persist data in the PostgreSQL ``plugins`` and ``revenue_events`` tables.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_current_user
|
||||
from app.db import get_session
|
||||
from app.marketplace.plugin_registry import registry
|
||||
from app.marketplace.revenue_share import revenue_share
|
||||
from app.models import PluginInstallation, PluginReview as PluginReviewModel
|
||||
from app.schemas import PluginInstallRequest, PluginListResponse, PluginManifest, UserProfile
|
||||
|
||||
router = APIRouter(prefix="/plugins", tags=["plugins"])
|
||||
|
||||
|
||||
# ── Tier gate ─────────────────────────────────────────────────────────
|
||||
|
||||
def _require_plugin_tier(user: UserProfile) -> None:
|
||||
"""Raise HTTP 403 for users below Power tier."""
|
||||
if user.tier not in ("power", "team"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Plugin marketplace requires Power tier or above",
|
||||
)
|
||||
|
||||
|
||||
# ── Local detail schema ────────────────────────────────────────────────
|
||||
|
||||
class _PluginDetail(BaseModel):
|
||||
plugin: PluginManifest
|
||||
install_count: int
|
||||
ratings: list[Any]
|
||||
|
||||
|
||||
# ── Routes ────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("", response_model=PluginListResponse)
|
||||
async def list_plugins(
|
||||
category: str | None = Query(default=None),
|
||||
q: str | None = Query(default=None),
|
||||
page: int = Query(default=1, ge=1),
|
||||
sort: Literal["rating", "installs", "newest"] = Query(default="newest"),
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> PluginListResponse:
|
||||
"""Browse the plugin marketplace. Requires Power tier or above."""
|
||||
_require_plugin_tier(current_user)
|
||||
return await registry.list_plugins(db, category=category, query=q, page=page, sort=sort)
|
||||
|
||||
|
||||
@router.get("/{plugin_id}", response_model=_PluginDetail)
|
||||
async def get_plugin(
|
||||
plugin_id: str,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> _PluginDetail:
|
||||
"""Get full plugin details including install count. Requires Power tier or above."""
|
||||
_require_plugin_tier(current_user)
|
||||
entry = await registry.get_plugin(db, plugin_id)
|
||||
if entry is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Plugin not found")
|
||||
|
||||
# Fetch review ratings for this plugin
|
||||
review_result = await db.execute(
|
||||
select(PluginReviewModel).where(PluginReviewModel.plugin_id == plugin_id)
|
||||
)
|
||||
reviews = review_result.scalars().all()
|
||||
ratings = [
|
||||
{
|
||||
"reviewer_id": r.reviewer_id,
|
||||
"decision": r.decision,
|
||||
"notes": r.notes,
|
||||
"reviewed_at": int(r.reviewed_at.timestamp() * 1000) if r.reviewed_at else None,
|
||||
}
|
||||
for r in reviews
|
||||
]
|
||||
|
||||
return _PluginDetail(
|
||||
plugin=entry["manifest"],
|
||||
install_count=entry["install_count"],
|
||||
ratings=ratings,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{plugin_id}/install", response_model=dict)
|
||||
async def install_plugin(
|
||||
plugin_id: str,
|
||||
body: PluginInstallRequest, # noqa: ARG001 — reserved for future fields
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> dict[str, Any]:
|
||||
"""Install a plugin. Triggers Stripe Connect revenue split for paid plugins.
|
||||
|
||||
Requires Power tier or above.
|
||||
"""
|
||||
_require_plugin_tier(current_user)
|
||||
entry = await registry.get_plugin(db, plugin_id)
|
||||
if entry is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Plugin not found")
|
||||
|
||||
# Record the installation in plugin_installations
|
||||
installation = PluginInstallation(
|
||||
plugin_id=plugin_id,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
db.add(installation)
|
||||
await db.flush()
|
||||
|
||||
await revenue_share.record_install(
|
||||
db,
|
||||
plugin_id=plugin_id,
|
||||
user_id=current_user.id,
|
||||
amount_cents=entry["manifest"].price_cents,
|
||||
)
|
||||
|
||||
download_url = f"https://cdn.adiuva.app/plugins/{plugin_id}/package.zip"
|
||||
return {"ok": True, "download_url": download_url}
|
||||
|
||||
|
||||
@router.delete("/{plugin_id}/install", response_model=dict)
|
||||
async def uninstall_plugin(
|
||||
plugin_id: str,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> dict[str, bool]:
|
||||
"""Unregister a plugin installation."""
|
||||
result = await db.execute(
|
||||
select(PluginInstallation).where(
|
||||
PluginInstallation.plugin_id == plugin_id,
|
||||
PluginInstallation.user_id == current_user.id,
|
||||
)
|
||||
)
|
||||
installation = result.scalar_one_or_none()
|
||||
if installation is not None:
|
||||
await db.delete(installation)
|
||||
await db.commit()
|
||||
await registry.record_uninstall(db, plugin_id)
|
||||
return {"ok": True}
|
||||
@@ -1,195 +0,0 @@
|
||||
"""Storage routes: CRUD for E2E-encrypted cloud records.
|
||||
|
||||
Blobs are stored in S3 via BlobStore. Record metadata is persisted in the
|
||||
PostgreSQL ``storage_records`` table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_current_user
|
||||
from app.billing.tier_manager import tier_manager
|
||||
from app.db import get_session
|
||||
from app.models import StorageRecord
|
||||
from app.schemas import StorageRecordCreate, StorageRecordUpdate, UserProfile
|
||||
from app.storage.blob_store import BlobStore
|
||||
from app.storage.encryption import reject_if_tampered
|
||||
|
||||
router = APIRouter(prefix="/storage", tags=["storage"])
|
||||
|
||||
_blob_store = BlobStore()
|
||||
|
||||
|
||||
# ── Local response schemas ─────────────────────────────────────────────
|
||||
|
||||
class _CreateResponse(BaseModel):
|
||||
id: str
|
||||
created_at: int
|
||||
|
||||
|
||||
class _RecordMeta(BaseModel):
|
||||
id: str
|
||||
table: str
|
||||
checksum: str
|
||||
created_at: int
|
||||
updated_at: int
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
async def _current_usage_bytes(user_id: str, db: AsyncSession) -> int:
|
||||
"""Return total bytes stored by *user_id*."""
|
||||
result = await db.execute(
|
||||
select(func.coalesce(func.sum(StorageRecord.size_bytes), 0)).where(
|
||||
StorageRecord.user_id == user_id
|
||||
)
|
||||
)
|
||||
return int(result.scalar_one())
|
||||
|
||||
|
||||
async def _check_quota(user: UserProfile, additional_bytes: int, db: AsyncSession) -> None:
|
||||
"""Raise HTTP 402 if adding *additional_bytes* would exceed the tier limit."""
|
||||
current = await _current_usage_bytes(user.id, db)
|
||||
tier_manager.enforce_quota(user.tier, current_bytes=current, additional_bytes=additional_bytes)
|
||||
|
||||
|
||||
async def _get_record_for_user(
|
||||
record_id: str, user_id: str, db: AsyncSession
|
||||
) -> StorageRecord:
|
||||
"""Look up a record and verify ownership. Returns 404 on mismatch
|
||||
to prevent user enumeration attacks."""
|
||||
result = await db.execute(
|
||||
select(StorageRecord).where(
|
||||
StorageRecord.id == record_id, StorageRecord.user_id == user_id
|
||||
)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
if record is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Record not found")
|
||||
return record
|
||||
|
||||
|
||||
# ── Routes ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/records", response_model=_CreateResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_record(
|
||||
body: StorageRecordCreate,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> _CreateResponse:
|
||||
"""Upload a new E2E-encrypted blob. Verifies checksum before storing."""
|
||||
reject_if_tampered(body.blob, body.checksum)
|
||||
await _check_quota(current_user, len(body.blob), db)
|
||||
|
||||
record_id = str(uuid.uuid4())
|
||||
|
||||
s3_key = await _blob_store.upload(
|
||||
current_user.id, body.table, record_id, body.blob, body.checksum
|
||||
)
|
||||
|
||||
record = StorageRecord(
|
||||
id=record_id,
|
||||
user_id=current_user.id,
|
||||
table_name=body.table,
|
||||
s3_key=s3_key,
|
||||
checksum=body.checksum,
|
||||
size_bytes=len(body.blob),
|
||||
)
|
||||
db.add(record)
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
|
||||
created_at_ms = int(record.created_at.timestamp() * 1000)
|
||||
return _CreateResponse(id=record_id, created_at=created_at_ms)
|
||||
|
||||
|
||||
@router.get("/records", response_model=list[_RecordMeta])
|
||||
async def list_records(
|
||||
table: str | None = Query(default=None),
|
||||
page: int = Query(default=1, ge=1),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> list[_RecordMeta]:
|
||||
"""List record metadata for the authenticated user. Blob bytes are never returned."""
|
||||
query = select(StorageRecord).where(StorageRecord.user_id == current_user.id)
|
||||
if table is not None:
|
||||
query = query.where(StorageRecord.table_name == table)
|
||||
query = query.offset((page - 1) * limit).limit(limit)
|
||||
|
||||
result = await db.execute(query)
|
||||
rows = result.scalars().all()
|
||||
|
||||
return [
|
||||
_RecordMeta(
|
||||
id=r.id,
|
||||
table=r.table_name,
|
||||
checksum=r.checksum,
|
||||
created_at=int(r.created_at.timestamp() * 1000),
|
||||
updated_at=int(r.updated_at.timestamp() * 1000),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("/records/{record_id}")
|
||||
async def download_record(
|
||||
record_id: str,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> Response:
|
||||
"""Download an E2E-encrypted blob. Returns raw bytes with ``X-Checksum`` header."""
|
||||
record = await _get_record_for_user(record_id, current_user.id, db)
|
||||
blob = await _blob_store.download(current_user.id, record.s3_key)
|
||||
return Response(
|
||||
content=blob,
|
||||
media_type="application/octet-stream",
|
||||
headers={"X-Checksum": record.checksum},
|
||||
)
|
||||
|
||||
|
||||
@router.put("/records/{record_id}", response_model=dict)
|
||||
async def update_record(
|
||||
record_id: str,
|
||||
body: StorageRecordUpdate,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> dict[str, bool]:
|
||||
"""Replace the blob for an existing record. Verifies checksum before storing."""
|
||||
record = await _get_record_for_user(record_id, current_user.id, db)
|
||||
reject_if_tampered(body.blob, body.checksum)
|
||||
|
||||
delta = len(body.blob) - record.size_bytes
|
||||
if delta > 0:
|
||||
await _check_quota(current_user, delta, db)
|
||||
|
||||
s3_key = await _blob_store.upload(
|
||||
current_user.id, record.table_name, record_id, body.blob, body.checksum
|
||||
)
|
||||
|
||||
record.s3_key = s3_key
|
||||
record.checksum = body.checksum
|
||||
record.size_bytes = len(body.blob)
|
||||
await db.commit()
|
||||
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.delete("/records/{record_id}", response_model=dict)
|
||||
async def delete_record(
|
||||
record_id: str,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_session),
|
||||
) -> dict[str, bool]:
|
||||
"""Delete a record and its S3 blob."""
|
||||
record = await _get_record_for_user(record_id, current_user.id, db)
|
||||
await _blob_store.delete(current_user.id, record.s3_key)
|
||||
await db.delete(record)
|
||||
await db.commit()
|
||||
return {"ok": True}
|
||||
@@ -1,79 +0,0 @@
|
||||
"""Vectors routes: upsert, search, delete cloud vector store entries, and embed text."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.api.deps import get_current_user
|
||||
from app.core.llm import embed
|
||||
from app.schemas import (
|
||||
UserProfile,
|
||||
VectorSearchRequest,
|
||||
VectorSearchResponse,
|
||||
VectorUpsertRequest,
|
||||
)
|
||||
from app.storage.encryption import reject_if_tampered
|
||||
from app.storage.vector_store import VectorStore
|
||||
|
||||
router = APIRouter(prefix="/storage", tags=["vectors"])
|
||||
|
||||
_vector_store = VectorStore()
|
||||
|
||||
|
||||
class _VectorDeleteRequest(BaseModel):
|
||||
ids: list[str]
|
||||
|
||||
|
||||
class _EmbedRequest(BaseModel):
|
||||
text: str
|
||||
|
||||
|
||||
class _EmbedResponse(BaseModel):
|
||||
vector: list[float]
|
||||
|
||||
|
||||
@router.post("/vectors/upsert", response_model=dict)
|
||||
async def upsert_vectors(
|
||||
body: VectorUpsertRequest,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
) -> dict[str, int]:
|
||||
"""Verify checksums and store encrypted vectors in the user-scoped namespace."""
|
||||
for item in body.vectors:
|
||||
reject_if_tampered(item.blob, item.checksum)
|
||||
await _vector_store.upsert(current_user.id, body.vectors)
|
||||
return {"upserted": len(body.vectors)}
|
||||
|
||||
|
||||
@router.post("/vectors/search", response_model=VectorSearchResponse)
|
||||
async def search_vectors(
|
||||
body: VectorSearchRequest,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
) -> VectorSearchResponse:
|
||||
"""Search the user-scoped vector namespace with an encrypted query blob."""
|
||||
results = await _vector_store.search(current_user.id, body.query_blob, body.top_k)
|
||||
return VectorSearchResponse(results=results)
|
||||
|
||||
|
||||
@router.delete("/vectors", response_model=dict)
|
||||
async def delete_vectors(
|
||||
body: _VectorDeleteRequest,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
) -> dict[str, bool]:
|
||||
"""Delete vectors by ID, scoped to the authenticated user."""
|
||||
await _vector_store.delete(current_user.id, body.ids)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/vectors/embed", response_model=_EmbedResponse)
|
||||
async def embed_text(
|
||||
body: _EmbedRequest,
|
||||
current_user: UserProfile = Depends(get_current_user),
|
||||
) -> _EmbedResponse:
|
||||
"""Generate a 1536-dim embedding vector for the given text.
|
||||
|
||||
Uses ``text-embedding-3-small`` via OpenAI. Auth required (JWT).
|
||||
Used by backend tools (note_agent) and Electron (vectordb.ts) alike.
|
||||
"""
|
||||
vector = await embed(body.text)
|
||||
return _EmbedResponse(vector=vector)
|
||||
@@ -22,44 +22,32 @@ FEATURES: dict[str, dict[str, Any]] = {
|
||||
"agents": 3,
|
||||
"batch_active": 2,
|
||||
"batch_runs_per_day": 5,
|
||||
"cloud_storage_gb": 0,
|
||||
"backup_gb": 0,
|
||||
"providers": 1,
|
||||
"batch_builder": False,
|
||||
"plugin_marketplace": False,
|
||||
"sso": False,
|
||||
},
|
||||
"pro": {
|
||||
"agents": -1, # unlimited
|
||||
"batch_active": 10,
|
||||
"batch_runs_per_day": 50,
|
||||
"cloud_storage_gb": 5,
|
||||
"backup_gb": 5,
|
||||
"providers": -1,
|
||||
"batch_builder": False,
|
||||
"plugin_marketplace": False,
|
||||
"sso": False,
|
||||
},
|
||||
"power": {
|
||||
"agents": -1,
|
||||
"batch_active": -1, # unlimited
|
||||
"batch_runs_per_day": -1, # unlimited
|
||||
"cloud_storage_gb": 25,
|
||||
"backup_gb": 25,
|
||||
"providers": -1,
|
||||
"batch_builder": True,
|
||||
"plugin_marketplace": True,
|
||||
"sso": False,
|
||||
},
|
||||
"team": {
|
||||
"agents": -1,
|
||||
"batch_active": -1,
|
||||
"batch_runs_per_day": -1, # unlimited
|
||||
"cloud_storage_gb": -1, # unlimited
|
||||
"backup_gb": -1, # unlimited
|
||||
"providers": -1,
|
||||
"batch_builder": True,
|
||||
"plugin_marketplace": True,
|
||||
"sso": True,
|
||||
},
|
||||
}
|
||||
@@ -125,71 +113,6 @@ class TierManager:
|
||||
"""Return the requests-per-minute limit for ``tier``."""
|
||||
return RATE_LIMITS.get(tier, RATE_LIMITS["free"])
|
||||
|
||||
# ── Storage quota ────────────────────────────────────────────────────
|
||||
|
||||
def enforce_quota(
|
||||
self,
|
||||
tier: BillingTier,
|
||||
current_bytes: int = 0,
|
||||
additional_bytes: int = 0,
|
||||
) -> None:
|
||||
"""Raise ``HTTP 402`` if the user would exceed their cloud storage quota.
|
||||
|
||||
``tier`` is the caller's current tier (from ``current_user.tier``).
|
||||
``current_bytes`` is the total bytes already stored (queried by caller).
|
||||
"""
|
||||
limit_gb: int = FEATURES[tier]["cloud_storage_gb"]
|
||||
if limit_gb == 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_402_PAYMENT_REQUIRED,
|
||||
detail=f"Cloud storage is not available on the '{tier}' tier",
|
||||
)
|
||||
if limit_gb == -1:
|
||||
return # unlimited
|
||||
limit_bytes = limit_gb * 1024 ** 3
|
||||
if current_bytes + additional_bytes > limit_bytes:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_402_PAYMENT_REQUIRED,
|
||||
detail=f"Storage quota exceeded for tier '{tier}'",
|
||||
)
|
||||
|
||||
def enforce_backup_quota(
|
||||
self,
|
||||
tier: BillingTier,
|
||||
current_bytes: int = 0,
|
||||
additional_bytes: int = 0,
|
||||
) -> None:
|
||||
"""Raise ``HTTP 402`` if the user would exceed their backup quota."""
|
||||
limit_gb: int = FEATURES[tier]["backup_gb"]
|
||||
if limit_gb == 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_402_PAYMENT_REQUIRED,
|
||||
detail=f"Backup is not available on the '{tier}' tier",
|
||||
)
|
||||
if limit_gb == -1:
|
||||
return # unlimited
|
||||
limit_bytes = limit_gb * 1024 ** 3
|
||||
if current_bytes + additional_bytes > limit_bytes:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_402_PAYMENT_REQUIRED,
|
||||
detail=f"Backup quota exceeded for tier '{tier}'",
|
||||
)
|
||||
|
||||
def check_quota(
|
||||
self,
|
||||
tier: BillingTier,
|
||||
current_bytes: int = 0,
|
||||
additional_bytes: int = 0,
|
||||
) -> bool:
|
||||
"""Return ``True`` if the user can store ``additional_bytes`` more data."""
|
||||
limit_gb: int = FEATURES[tier]["cloud_storage_gb"]
|
||||
if limit_gb == 0:
|
||||
return False
|
||||
if limit_gb == -1:
|
||||
return True
|
||||
limit_bytes = limit_gb * 1024 ** 3
|
||||
return current_bytes + additional_bytes <= limit_bytes
|
||||
|
||||
|
||||
# Module-level singleton shared across the app.
|
||||
tier_manager = TierManager()
|
||||
|
||||
@@ -12,17 +12,6 @@ class Settings(BaseSettings):
|
||||
STRIPE_SECRET_KEY: str = ""
|
||||
STRIPE_WEBHOOK_SECRET: str = ""
|
||||
|
||||
S3_BUCKET: str = ""
|
||||
S3_REGION: str = "us-east-1"
|
||||
S3_ENDPOINT_URL: str = ""
|
||||
AWS_ACCESS_KEY_ID: str = ""
|
||||
AWS_SECRET_ACCESS_KEY: str = ""
|
||||
|
||||
PINECONE_API_KEY: str = ""
|
||||
PINECONE_INDEX: str = "adiuva"
|
||||
QDRANT_URL: str = ""
|
||||
QDRANT_API_KEY: str = ""
|
||||
|
||||
OPENAI_API_KEY: str = ""
|
||||
ANTHROPIC_API_KEY: str = ""
|
||||
GOOGLE_API_KEY: str = ""
|
||||
|
||||
@@ -50,14 +50,10 @@ def create_app() -> FastAPI:
|
||||
app.add_middleware(SanitizerMiddleware)
|
||||
app.add_middleware(TierRateLimitMiddleware)
|
||||
|
||||
from app.api.routes import agents, auth, backup, billing, chat, device_ws, plugins, storage, vectors
|
||||
from app.api.routes import agents, auth, billing, chat, device_ws
|
||||
|
||||
app.include_router(auth.router, prefix="/api/v1")
|
||||
app.include_router(chat.router, prefix="/api/v1")
|
||||
app.include_router(storage.router, prefix="/api/v1")
|
||||
app.include_router(vectors.router, prefix="/api/v1")
|
||||
app.include_router(backup.router, prefix="/api/v1")
|
||||
app.include_router(plugins.router, prefix="/api/v1")
|
||||
app.include_router(billing.router, prefix="/api/v1")
|
||||
app.include_router(agents.router, prefix="/api/v1")
|
||||
app.include_router(device_ws.router, prefix="/api/v1")
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
"""Plugin marketplace package.
|
||||
|
||||
Three service classes introduced in Step 10:
|
||||
- ``PluginRegistry`` — catalog, submit/approve/reject, install counts
|
||||
- ``ReviewQueue`` — approval workflow + security checklist
|
||||
- ``RevenueShare`` — 70/30 split tracking and Stripe Connect payouts
|
||||
"""
|
||||
@@ -1,212 +0,0 @@
|
||||
"""Plugin catalog registry backed by PostgreSQL.
|
||||
|
||||
Maintains the authoritative list of plugins, their review status, and
|
||||
aggregate install counts. All data is persisted in the ``plugins`` table.
|
||||
|
||||
Module-level singleton::
|
||||
|
||||
from app.marketplace.plugin_registry import registry
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Literal
|
||||
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Plugin
|
||||
from app.schemas import PluginListResponse, PluginManifest
|
||||
|
||||
_PAGE_SIZE = 20
|
||||
|
||||
|
||||
def _plugin_to_manifest(p: Plugin) -> PluginManifest:
|
||||
"""Convert an ORM ``Plugin`` row to a Pydantic ``PluginManifest``."""
|
||||
try:
|
||||
permissions = json.loads(p.permissions) if p.permissions else []
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
permissions = []
|
||||
return PluginManifest(
|
||||
id=p.id,
|
||||
name=p.name,
|
||||
description=p.description,
|
||||
version=p.version,
|
||||
author=p.author_name,
|
||||
permissions=permissions,
|
||||
category=p.category,
|
||||
price_cents=p.price_cents,
|
||||
)
|
||||
|
||||
|
||||
class PluginRegistry:
|
||||
"""PostgreSQL-backed plugin catalog.
|
||||
|
||||
All methods accept an ``AsyncSession`` parameter so the calling route
|
||||
controls the session lifecycle.
|
||||
"""
|
||||
|
||||
# ── Queries ──────────────────────────────────────────────────────
|
||||
|
||||
async def list_plugins(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
category: str | None = None,
|
||||
query: str | None = None,
|
||||
page: int = 1,
|
||||
sort: Literal["rating", "installs", "newest"] = "newest",
|
||||
) -> PluginListResponse:
|
||||
"""Return a page of approved plugins, optionally filtered and sorted."""
|
||||
base = select(Plugin).where(Plugin.status == "approved")
|
||||
|
||||
if category:
|
||||
base = base.where(Plugin.category == category)
|
||||
if query:
|
||||
pattern = f"%{query}%"
|
||||
base = base.where(
|
||||
Plugin.name.ilike(pattern) | Plugin.description.ilike(pattern)
|
||||
)
|
||||
|
||||
# Count
|
||||
count_q = select(func.count()).select_from(base.subquery())
|
||||
total = (await db.execute(count_q)).scalar_one()
|
||||
|
||||
# Sort
|
||||
if sort == "installs":
|
||||
base = base.order_by(Plugin.install_count.desc())
|
||||
elif sort == "rating":
|
||||
base = base.order_by(Plugin.avg_rating.desc())
|
||||
else: # newest
|
||||
base = base.order_by(Plugin.created_at.desc())
|
||||
|
||||
base = base.offset((page - 1) * _PAGE_SIZE).limit(_PAGE_SIZE)
|
||||
rows = (await db.execute(base)).scalars().all()
|
||||
|
||||
return PluginListResponse(
|
||||
plugins=[_plugin_to_manifest(r) for r in rows],
|
||||
total=total,
|
||||
page=page,
|
||||
)
|
||||
|
||||
async def get_plugin(self, db: AsyncSession, plugin_id: str) -> dict[str, Any] | None:
|
||||
"""Return ``{manifest, status, install_count, avg_rating}`` or ``None``."""
|
||||
result = await db.execute(select(Plugin).where(Plugin.id == plugin_id))
|
||||
p = result.scalar_one_or_none()
|
||||
if p is None:
|
||||
return None
|
||||
return {
|
||||
"manifest": _plugin_to_manifest(p),
|
||||
"status": p.status,
|
||||
"install_count": p.install_count,
|
||||
"avg_rating": p.avg_rating,
|
||||
}
|
||||
|
||||
# ── Mutations ────────────────────────────────────────────────────
|
||||
|
||||
async def submit_plugin(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
manifest: PluginManifest,
|
||||
package_s3_key: str,
|
||||
) -> str:
|
||||
"""Add *manifest* to the catalog with ``status='pending_review'``.
|
||||
|
||||
Returns the plugin_id. If a plugin with the same id already exists
|
||||
it is overwritten (re-submission after rejection).
|
||||
"""
|
||||
plugin_id = manifest.id
|
||||
existing = await db.execute(select(Plugin).where(Plugin.id == plugin_id))
|
||||
row = existing.scalar_one_or_none()
|
||||
|
||||
if row is not None:
|
||||
row.name = manifest.name
|
||||
row.description = manifest.description
|
||||
row.version = manifest.version
|
||||
row.author_name = manifest.author
|
||||
row.category = manifest.category
|
||||
row.price_cents = manifest.price_cents
|
||||
row.permissions = json.dumps(manifest.permissions)
|
||||
row.status = "pending_review"
|
||||
row.s3_package_key = package_s3_key
|
||||
row.rejection_reason = None
|
||||
else:
|
||||
row = Plugin(
|
||||
id=plugin_id,
|
||||
name=manifest.name,
|
||||
description=manifest.description,
|
||||
version=manifest.version,
|
||||
author_name=manifest.author,
|
||||
category=manifest.category,
|
||||
price_cents=manifest.price_cents,
|
||||
permissions=json.dumps(manifest.permissions),
|
||||
status="pending_review",
|
||||
s3_package_key=package_s3_key,
|
||||
install_count=0,
|
||||
avg_rating=0.0,
|
||||
)
|
||||
db.add(row)
|
||||
await db.commit()
|
||||
return plugin_id
|
||||
|
||||
async def approve_plugin(self, db: AsyncSession, plugin_id: str) -> None:
|
||||
"""Set *plugin_id* status to ``'approved'``.
|
||||
|
||||
Raises ``KeyError`` if the plugin is not found.
|
||||
"""
|
||||
result = await db.execute(select(Plugin).where(Plugin.id == plugin_id))
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
raise KeyError(f"Plugin not found: {plugin_id}")
|
||||
row.status = "approved"
|
||||
row.rejection_reason = None
|
||||
await db.commit()
|
||||
|
||||
async def reject_plugin(self, db: AsyncSession, plugin_id: str, reason: str) -> None:
|
||||
"""Set *plugin_id* status to ``'rejected'`` and record the reason.
|
||||
|
||||
Raises ``KeyError`` if the plugin is not found.
|
||||
"""
|
||||
result = await db.execute(select(Plugin).where(Plugin.id == plugin_id))
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
raise KeyError(f"Plugin not found: {plugin_id}")
|
||||
row.status = "rejected"
|
||||
row.rejection_reason = reason
|
||||
await db.commit()
|
||||
|
||||
async def record_install(self, db: AsyncSession, plugin_id: str) -> None:
|
||||
"""Increment the install count for *plugin_id* (no-op if not found)."""
|
||||
result = await db.execute(select(Plugin).where(Plugin.id == plugin_id))
|
||||
row = result.scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.install_count = row.install_count + 1
|
||||
await db.commit()
|
||||
|
||||
async def record_uninstall(self, db: AsyncSession, plugin_id: str) -> None:
|
||||
"""Decrement the install count for *plugin_id*, floored at 0."""
|
||||
result = await db.execute(select(Plugin).where(Plugin.id == plugin_id))
|
||||
row = result.scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.install_count = max(0, row.install_count - 1)
|
||||
await db.commit()
|
||||
|
||||
# ── Internal helpers used by ReviewQueue ─────────────────────────
|
||||
|
||||
async def get_pending_entries(self, db: AsyncSession) -> list[dict[str, Any]]:
|
||||
"""Return all entries with status='pending_review'."""
|
||||
result = await db.execute(
|
||||
select(Plugin).where(Plugin.status == "pending_review")
|
||||
)
|
||||
rows = result.scalars().all()
|
||||
return [
|
||||
{
|
||||
"manifest": _plugin_to_manifest(r),
|
||||
"submitted_at": int(r.submitted_at.timestamp()) if r.submitted_at else 0,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
# Module-level singleton
|
||||
registry = PluginRegistry()
|
||||
@@ -1,125 +0,0 @@
|
||||
"""Plugin review workflow backed by PostgreSQL.
|
||||
|
||||
Manages the approval queue for newly submitted plugins and enforces a
|
||||
security checklist before any plugin is made visible in the marketplace.
|
||||
|
||||
Module-level singleton::
|
||||
|
||||
from app.marketplace.plugin_review import review_queue
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Literal
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.marketplace.plugin_registry import registry
|
||||
from app.models import PluginReview as PluginReviewModel
|
||||
from app.schemas import PluginManifest
|
||||
|
||||
# ── Security policy ───────────────────────────────────────────────────
|
||||
|
||||
ALLOWED_PERMISSIONS: frozenset[str] = frozenset(
|
||||
{
|
||||
"read:tasks",
|
||||
"write:tasks",
|
||||
"read:projects",
|
||||
"write:projects",
|
||||
"read:notes",
|
||||
"write:notes",
|
||||
"read:timelines",
|
||||
"write:timelines",
|
||||
"read:calendar",
|
||||
"write:calendar",
|
||||
}
|
||||
)
|
||||
|
||||
_PLUGIN_ID_RE = re.compile(r"^[a-z0-9-]+$")
|
||||
|
||||
|
||||
def validate_manifest(manifest: PluginManifest) -> None:
|
||||
"""Enforce the plugin security checklist.
|
||||
|
||||
Raises:
|
||||
``ValueError`` on the first violation found. Callers should catch
|
||||
this and return HTTP 422 / reject the submission.
|
||||
|
||||
Checks:
|
||||
1. Plugin id matches ``^[a-z0-9-]+$``
|
||||
2. All declared permissions are in ``ALLOWED_PERMISSIONS``
|
||||
3. No manifest field contains raw binary data
|
||||
"""
|
||||
if not _PLUGIN_ID_RE.match(manifest.id):
|
||||
raise ValueError(
|
||||
f"Invalid plugin id format: '{manifest.id}'. "
|
||||
"Only lowercase letters, digits, and hyphens are allowed."
|
||||
)
|
||||
|
||||
for perm in manifest.permissions:
|
||||
if perm not in ALLOWED_PERMISSIONS:
|
||||
raise ValueError(
|
||||
f"Unknown permission: '{perm}'. "
|
||||
f"Allowed permissions: {sorted(ALLOWED_PERMISSIONS)}"
|
||||
)
|
||||
|
||||
for field_name, value in manifest.model_dump().items():
|
||||
if isinstance(value, (bytes, bytearray)):
|
||||
raise ValueError(
|
||||
f"Binary content is not allowed in manifest field '{field_name}'."
|
||||
)
|
||||
|
||||
|
||||
class ReviewQueue:
|
||||
"""Approval queue for pending plugin submissions.
|
||||
|
||||
Delegates status changes to the shared ``PluginRegistry`` singleton.
|
||||
Review records are persisted in the ``plugin_reviews`` table.
|
||||
"""
|
||||
|
||||
async def get_pending(self, db: AsyncSession) -> list[dict[str, Any]]:
|
||||
"""Return all plugins currently awaiting review.
|
||||
|
||||
Each item is ``{plugin_id, manifest, submitted_at}``.
|
||||
"""
|
||||
entries = await registry.get_pending_entries(db)
|
||||
return [
|
||||
{
|
||||
"plugin_id": e["manifest"].id,
|
||||
"manifest": e["manifest"],
|
||||
"submitted_at": e["submitted_at"],
|
||||
}
|
||||
for e in entries
|
||||
]
|
||||
|
||||
async def submit_review(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
plugin_id: str,
|
||||
reviewer_id: str,
|
||||
decision: Literal["approved", "rejected"],
|
||||
notes: str = "",
|
||||
) -> None:
|
||||
"""Record a review decision and update the plugin's status.
|
||||
|
||||
Raises:
|
||||
``KeyError`` if *plugin_id* is not found in the registry.
|
||||
"""
|
||||
if decision == "approved":
|
||||
await registry.approve_plugin(db, plugin_id)
|
||||
else:
|
||||
await registry.reject_plugin(db, plugin_id, reason=notes)
|
||||
|
||||
review = PluginReviewModel(
|
||||
plugin_id=plugin_id,
|
||||
reviewer_id=reviewer_id,
|
||||
decision=decision,
|
||||
notes=notes,
|
||||
)
|
||||
db.add(review)
|
||||
await db.commit()
|
||||
|
||||
|
||||
# Module-level singleton
|
||||
review_queue = ReviewQueue()
|
||||
@@ -1,233 +0,0 @@
|
||||
"""Revenue share tracking and Stripe Connect payouts backed by PostgreSQL.
|
||||
|
||||
Records every plugin installation as a revenue event and facilitates
|
||||
70 % / 30 % payouts to developers via Stripe Connect. Data is persisted
|
||||
in the ``revenue_events`` table.
|
||||
|
||||
Module-level singleton::
|
||||
|
||||
from app.marketplace.revenue_share import revenue_share
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import stripe as stripe_lib
|
||||
from sqlalchemy import extract, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.marketplace.plugin_registry import registry
|
||||
from app.models import Plugin, RevenueEvent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Revenue split constants ───────────────────────────────────────────
|
||||
|
||||
DEVELOPER_SHARE: float = 0.70
|
||||
PLATFORM_SHARE: float = 0.30
|
||||
|
||||
|
||||
class RevenueShare:
|
||||
"""Records installation revenue events and coordinates developer payouts.
|
||||
|
||||
Stripe Connect calls are gracefully stubbed when ``STRIPE_SECRET_KEY``
|
||||
is not configured, consistent with the rest of the billing layer.
|
||||
"""
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _stripe_configured() -> bool:
|
||||
return bool(settings.STRIPE_SECRET_KEY)
|
||||
|
||||
@staticmethod
|
||||
def _stripe() -> Any:
|
||||
stripe_lib.api_key = settings.STRIPE_SECRET_KEY
|
||||
return stripe_lib
|
||||
|
||||
# ── Core operations ──────────────────────────────────────────────
|
||||
|
||||
async def record_install(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
plugin_id: str,
|
||||
user_id: str,
|
||||
amount_cents: int,
|
||||
) -> None:
|
||||
"""Record a plugin installation and trigger a Stripe Connect charge if paid.
|
||||
|
||||
For free plugins (``amount_cents == 0``) no payment is initiated but
|
||||
the event is still recorded for analytics.
|
||||
|
||||
For paid plugins the developer receives 70 % via a Stripe Connect
|
||||
destination charge. If Stripe is not configured or the charge fails
|
||||
the installation still succeeds (the event is recorded and the install
|
||||
count is incremented) — a warning is logged for monitoring.
|
||||
"""
|
||||
developer_share_cents = int(amount_cents * DEVELOPER_SHARE)
|
||||
stripe_transfer_id: str | None = None
|
||||
|
||||
if amount_cents > 0 and self._stripe_configured():
|
||||
# Look up the plugin's author Stripe account from the DB
|
||||
result = await db.execute(select(Plugin).where(Plugin.id == plugin_id))
|
||||
plugin_row = result.scalar_one_or_none()
|
||||
developer_stripe_account: str | None = None
|
||||
if plugin_row and plugin_row.author_id:
|
||||
# Future: look up user.stripe_connect_account_id
|
||||
developer_stripe_account = None # no real account yet
|
||||
|
||||
if developer_stripe_account:
|
||||
try:
|
||||
s = self._stripe()
|
||||
transfer = s.Transfer.create(
|
||||
amount=developer_share_cents,
|
||||
currency="eur",
|
||||
destination=developer_stripe_account,
|
||||
description=f"Revenue share for plugin {plugin_id}",
|
||||
metadata={"plugin_id": plugin_id, "user_id": user_id},
|
||||
)
|
||||
stripe_transfer_id = transfer["id"]
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Stripe Connect transfer failed for plugin %s: %s",
|
||||
plugin_id,
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"No Stripe account on file for plugin %s developer; "
|
||||
"skipping transfer.",
|
||||
plugin_id,
|
||||
)
|
||||
|
||||
event = RevenueEvent(
|
||||
plugin_id=plugin_id,
|
||||
user_id=user_id,
|
||||
amount_cents=amount_cents,
|
||||
developer_share_cents=developer_share_cents,
|
||||
stripe_transfer_id=stripe_transfer_id,
|
||||
)
|
||||
db.add(event)
|
||||
await db.commit()
|
||||
|
||||
await registry.record_install(db, plugin_id)
|
||||
|
||||
async def get_earnings(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
developer_id: str,
|
||||
period: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return aggregated earnings for *developer_id*.
|
||||
|
||||
``period`` is an optional ``YYYY-MM`` string to restrict the window.
|
||||
|
||||
Returns::
|
||||
|
||||
{
|
||||
"developer_id": str,
|
||||
"period": str | None,
|
||||
"total_installs": int,
|
||||
"total_revenue_cents": int,
|
||||
"developer_share_cents": int,
|
||||
}
|
||||
"""
|
||||
# Find plugin ids belonging to this developer (by author_name match)
|
||||
plugin_q = select(Plugin.id).where(Plugin.author_name == developer_id)
|
||||
plugin_result = await db.execute(plugin_q)
|
||||
developer_plugin_ids = [row[0] for row in plugin_result.all()]
|
||||
|
||||
if not developer_plugin_ids:
|
||||
return {
|
||||
"developer_id": developer_id,
|
||||
"period": period,
|
||||
"total_installs": 0,
|
||||
"total_revenue_cents": 0,
|
||||
"developer_share_cents": 0,
|
||||
}
|
||||
|
||||
query = select(
|
||||
func.count().label("total_installs"),
|
||||
func.coalesce(func.sum(RevenueEvent.amount_cents), 0).label("total_revenue"),
|
||||
func.coalesce(func.sum(RevenueEvent.developer_share_cents), 0).label("dev_share"),
|
||||
).where(RevenueEvent.plugin_id.in_(developer_plugin_ids))
|
||||
|
||||
if period:
|
||||
# Filter by YYYY-MM: extract year and month from created_at
|
||||
try:
|
||||
year, month = period.split("-")
|
||||
query = query.where(
|
||||
extract("year", RevenueEvent.created_at) == int(year),
|
||||
extract("month", RevenueEvent.created_at) == int(month),
|
||||
)
|
||||
except ValueError:
|
||||
pass # invalid period format — return all
|
||||
|
||||
result = await db.execute(query)
|
||||
row = result.one()
|
||||
|
||||
return {
|
||||
"developer_id": developer_id,
|
||||
"period": period,
|
||||
"total_installs": row.total_installs,
|
||||
"total_revenue_cents": row.total_revenue,
|
||||
"developer_share_cents": row.dev_share,
|
||||
}
|
||||
|
||||
async def payout_developer(self, db: AsyncSession, plugin_id: str, period: str) -> None:
|
||||
"""Aggregate unpaid revenue for *period* and issue a Stripe Transfer.
|
||||
|
||||
Marks processed events with ``paid_at`` timestamp.
|
||||
Stubs gracefully when Stripe is not configured.
|
||||
"""
|
||||
try:
|
||||
year, month = period.split("-")
|
||||
year_int, month_int = int(year), int(month)
|
||||
except ValueError:
|
||||
logger.warning("Invalid period format: %s", period)
|
||||
return
|
||||
|
||||
result = await db.execute(
|
||||
select(RevenueEvent).where(
|
||||
RevenueEvent.plugin_id == plugin_id,
|
||||
RevenueEvent.paid_at.is_(None),
|
||||
extract("year", RevenueEvent.created_at) == year_int,
|
||||
extract("month", RevenueEvent.created_at) == month_int,
|
||||
)
|
||||
)
|
||||
unpaid = list(result.scalars().all())
|
||||
|
||||
total_dev_share = sum(e.developer_share_cents for e in unpaid)
|
||||
if total_dev_share <= 0 or not unpaid:
|
||||
logger.debug("Nothing to pay out for plugin %s in period %s", plugin_id, period)
|
||||
return
|
||||
|
||||
if self._stripe_configured():
|
||||
plugin_result = await db.execute(select(Plugin).where(Plugin.id == plugin_id))
|
||||
plugin_row = plugin_result.scalar_one_or_none()
|
||||
developer_stripe_account: str | None = None # Future: fetch from DB
|
||||
if plugin_row and developer_stripe_account:
|
||||
try:
|
||||
s = self._stripe()
|
||||
s.Transfer.create(
|
||||
amount=total_dev_share,
|
||||
currency="eur",
|
||||
destination=developer_stripe_account,
|
||||
description=f"Payout for plugin {plugin_id} period {period}",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Payout transfer failed for plugin %s: %s", plugin_id, exc)
|
||||
return
|
||||
|
||||
paid_ts = datetime.now(timezone.utc)
|
||||
for event in unpaid:
|
||||
event.paid_at = paid_ts
|
||||
await db.commit()
|
||||
|
||||
|
||||
# Module-level singleton
|
||||
revenue_share = RevenueShare()
|
||||
163
app/models.py
163
app/models.py
@@ -1,19 +1,15 @@
|
||||
"""SQLAlchemy ORM models for all persistent tables.
|
||||
|
||||
Only auth, billing, storage metadata, and marketplace data live here.
|
||||
User content (notes, tasks, etc.) is NEVER persisted server-side —
|
||||
it lives in E2E-encrypted blobs in S3, referenced by storage_records.
|
||||
Only auth, billing, agent config, and memory data live here.
|
||||
User content (notes, tasks, etc.) lives exclusively on the client.
|
||||
|
||||
Table inventory:
|
||||
users — account credentials + tier
|
||||
refresh_tokens — hashed refresh token store
|
||||
subscriptions — Stripe subscription records
|
||||
storage_records — S3 blob metadata (no plaintext)
|
||||
backup_metadata — encrypted backup manifests
|
||||
plugins — marketplace plugin catalog
|
||||
plugin_installations — per-user install records
|
||||
plugin_reviews — admin review decisions
|
||||
revenue_events — Stripe Connect 70/30 split ledger
|
||||
local_agent_configs — per-device batch agent configs
|
||||
cloud_agent_configs — OAuth-backed cloud agent configs
|
||||
agent_run_logs — execution history for all agents
|
||||
memory_core — per-user persistent key/value preferences (encrypted)
|
||||
memory_associative — per-user semantic memory with embeddings (encrypted)
|
||||
memory_episodic — per-user session summaries (encrypted)
|
||||
@@ -26,7 +22,6 @@ import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
DateTime,
|
||||
Enum,
|
||||
@@ -36,7 +31,6 @@ from sqlalchemy import (
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
Uuid,
|
||||
func,
|
||||
)
|
||||
@@ -58,8 +52,6 @@ def _now() -> datetime:
|
||||
# ── Enum types ────────────────────────────────────────────────────────────
|
||||
|
||||
TierEnum = Enum("free", "pro", "power", "team", name="billing_tier")
|
||||
PluginStatusEnum = Enum("pending_review", "approved", "rejected", name="plugin_status")
|
||||
ReviewDecisionEnum = Enum("approved", "rejected", name="review_decision")
|
||||
AgentTypeEnum = Enum("local", "cloud", name="agent_type")
|
||||
AgentStatusEnum = Enum("running", "success", "error", "partial", name="agent_run_status")
|
||||
CloudProviderEnum = Enum("gmail", "teams", "outlook", name="cloud_provider")
|
||||
@@ -137,151 +129,6 @@ class Subscription(Base):
|
||||
user: Mapped[User] = relationship(back_populates="subscription")
|
||||
|
||||
|
||||
class StorageRecord(Base):
|
||||
__tablename__ = "storage_records"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), primary_key=True, default=_uuid
|
||||
)
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
table_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
s3_key: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
checksum: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
|
||||
|
||||
class BackupMetadata(Base):
|
||||
__tablename__ = "backup_metadata"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), primary_key=True, default=_uuid
|
||||
)
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
s3_key: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
timestamp: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
checksum: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class Plugin(Base):
|
||||
__tablename__ = "plugins"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
description: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
version: Mapped[str] = mapped_column(String(50), nullable=False, default="1.0.0")
|
||||
# nullable until developer account system is built
|
||||
author_id: Mapped[str | None] = mapped_column(
|
||||
Uuid(as_uuid=False), ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
author_name: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
category: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||
price_cents: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
permissions: Mapped[str] = mapped_column(Text, nullable=False, default="[]") # JSON list
|
||||
status: Mapped[str] = mapped_column(PluginStatusEnum, nullable=False, default="pending_review")
|
||||
s3_package_key: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
install_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
avg_rating: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
rejection_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
submitted_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
installations: Mapped[list[PluginInstallation]] = relationship(
|
||||
back_populates="plugin", cascade="all, delete-orphan"
|
||||
)
|
||||
reviews: Mapped[list[PluginReview]] = relationship(
|
||||
back_populates="plugin", cascade="all, delete-orphan"
|
||||
)
|
||||
revenue_events: Mapped[list[RevenueEvent]] = relationship(
|
||||
back_populates="plugin", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
class PluginInstallation(Base):
|
||||
__tablename__ = "plugin_installations"
|
||||
__table_args__ = (UniqueConstraint("plugin_id", "user_id", name="uq_plugin_user"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), primary_key=True, default=_uuid
|
||||
)
|
||||
plugin_id: Mapped[str] = mapped_column(
|
||||
String(255), ForeignKey("plugins.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
installed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
plugin: Mapped[Plugin] = relationship(back_populates="installations")
|
||||
|
||||
|
||||
class PluginReview(Base):
|
||||
__tablename__ = "plugin_reviews"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), primary_key=True, default=_uuid
|
||||
)
|
||||
plugin_id: Mapped[str] = mapped_column(
|
||||
String(255), ForeignKey("plugins.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
reviewer_id: Mapped[str | None] = mapped_column(
|
||||
Uuid(as_uuid=False), ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
decision: Mapped[str] = mapped_column(ReviewDecisionEnum, nullable=False)
|
||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
reviewed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
plugin: Mapped[Plugin] = relationship(back_populates="reviews")
|
||||
|
||||
|
||||
class RevenueEvent(Base):
|
||||
__tablename__ = "revenue_events"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), primary_key=True, default=_uuid
|
||||
)
|
||||
plugin_id: Mapped[str] = mapped_column(
|
||||
String(255), ForeignKey("plugins.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
user_id: Mapped[str] = mapped_column(
|
||||
Uuid(as_uuid=False), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
amount_cents: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
developer_share_cents: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
stripe_transfer_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
paid_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
plugin: Mapped[Plugin] = relationship(back_populates="revenue_events")
|
||||
|
||||
|
||||
class LocalAgentConfig(Base):
|
||||
__tablename__ = "local_agent_configs"
|
||||
|
||||
|
||||
@@ -50,88 +50,6 @@ class ChatResponse(BaseModel):
|
||||
response: str
|
||||
|
||||
|
||||
# ── Backup ───────────────────────────────────────────────────────────
|
||||
|
||||
class BackupMetadata(BaseModel):
|
||||
version: int
|
||||
timestamp: int
|
||||
checksum: str
|
||||
chunk_count: int
|
||||
|
||||
|
||||
# ── Cloud Storage (E2E encrypted blobs) ──────────────────────────────
|
||||
|
||||
class StorageRecord(BaseModel):
|
||||
id: str
|
||||
user_id: str
|
||||
table: str
|
||||
blob: bytes
|
||||
checksum: str
|
||||
created_at: int
|
||||
updated_at: int
|
||||
|
||||
|
||||
class StorageRecordCreate(BaseModel):
|
||||
table: str
|
||||
blob: bytes
|
||||
checksum: str
|
||||
|
||||
|
||||
class StorageRecordUpdate(BaseModel):
|
||||
blob: bytes
|
||||
checksum: str
|
||||
|
||||
|
||||
# ── Cloud Vector Store (E2E encrypted vectors) ────────────────────────
|
||||
|
||||
class VectorItem(BaseModel):
|
||||
id: str
|
||||
blob: bytes # encrypted vector + metadata — backend never decrypts
|
||||
checksum: str
|
||||
|
||||
|
||||
class VectorUpsertRequest(BaseModel):
|
||||
vectors: list[VectorItem]
|
||||
|
||||
|
||||
class VectorSearchRequest(BaseModel):
|
||||
query_blob: bytes # encrypted query — backend never decrypts
|
||||
top_k: int = 10
|
||||
|
||||
|
||||
class VectorSearchResult(BaseModel):
|
||||
id: str
|
||||
score: float
|
||||
blob: bytes
|
||||
|
||||
|
||||
class VectorSearchResponse(BaseModel):
|
||||
results: list[VectorSearchResult]
|
||||
|
||||
|
||||
# ── Plugin Marketplace ────────────────────────────────────────────────
|
||||
|
||||
class PluginManifest(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
description: str
|
||||
version: str
|
||||
author: str
|
||||
permissions: list[str]
|
||||
category: str
|
||||
price_cents: int = 0
|
||||
|
||||
|
||||
class PluginListResponse(BaseModel):
|
||||
plugins: list[PluginManifest]
|
||||
total: int
|
||||
page: int
|
||||
|
||||
|
||||
class PluginInstallRequest(BaseModel):
|
||||
plugin_id: str
|
||||
|
||||
|
||||
# ── WebSocket Frame Protocol ──────────────────────────────────────────
|
||||
|
||||
class WsFrameType(str, Enum):
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""Cloud storage layer — E2E encrypted blobs and vectors."""
|
||||
@@ -1,106 +0,0 @@
|
||||
"""S3-backed store for E2E-encrypted blobs.
|
||||
|
||||
Keys are structured as ``{user_id}/{table}/{record_id}``.
|
||||
The backend never inspects blob content — it stores and retrieves opaque bytes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import boto3
|
||||
|
||||
from app.config.settings import settings
|
||||
|
||||
|
||||
class BlobStore:
|
||||
"""Thin wrapper around boto3 S3.
|
||||
|
||||
All blobs must be E2E encrypted by the client before upload.
|
||||
The backend adds SSE-S3 as an extra layer of at-rest encryption
|
||||
but cannot decrypt the inner client-side payload.
|
||||
"""
|
||||
|
||||
def _client(self) -> Any:
|
||||
kwargs: dict[str, Any] = {
|
||||
"region_name": settings.S3_REGION,
|
||||
"aws_access_key_id": settings.AWS_ACCESS_KEY_ID,
|
||||
"aws_secret_access_key": settings.AWS_SECRET_ACCESS_KEY,
|
||||
}
|
||||
if settings.S3_ENDPOINT_URL and isinstance(settings.S3_ENDPOINT_URL, str):
|
||||
kwargs["endpoint_url"] = settings.S3_ENDPOINT_URL
|
||||
return boto3.client("s3", **kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _key(user_id: str, table: str, record_id: str) -> str:
|
||||
return f"{user_id}/{table}/{record_id}"
|
||||
|
||||
async def upload(
|
||||
self,
|
||||
user_id: str,
|
||||
table: str,
|
||||
record_id: str,
|
||||
blob: bytes,
|
||||
checksum: str,
|
||||
) -> str:
|
||||
"""Store *blob* in S3 and return the S3 key.
|
||||
|
||||
Args:
|
||||
user_id: Owner of the blob (used as key prefix).
|
||||
table: Logical table name (e.g. ``"tasks"``).
|
||||
record_id: Record UUID.
|
||||
blob: Raw bytes (pre-encrypted by client).
|
||||
checksum: SHA-256 hex digest supplied by the client; stored as
|
||||
object metadata for download-time verification.
|
||||
|
||||
Returns:
|
||||
The S3 key under which the blob was stored.
|
||||
"""
|
||||
key = self._key(user_id, table, record_id)
|
||||
self._client().put_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=key,
|
||||
Body=blob,
|
||||
ServerSideEncryption="AES256", # SSE-S3 at rest
|
||||
Metadata={"checksum": checksum},
|
||||
)
|
||||
return key
|
||||
|
||||
async def download(self, user_id: str, s3_key: str) -> bytes:
|
||||
"""Retrieve the blob stored at *s3_key*.
|
||||
|
||||
*user_id* is retained in the signature so higher-level code can
|
||||
enforce ownership without re-parsing the key.
|
||||
|
||||
Raises:
|
||||
``botocore.exceptions.ClientError`` with code ``NoSuchKey`` if the
|
||||
object does not exist.
|
||||
"""
|
||||
response = self._client().get_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=s3_key,
|
||||
)
|
||||
return response["Body"].read()
|
||||
|
||||
async def delete(self, user_id: str, s3_key: str) -> None:
|
||||
"""Delete the object at *s3_key*.
|
||||
|
||||
S3 ``delete_object`` is idempotent — it succeeds even if the key does
|
||||
not exist.
|
||||
"""
|
||||
self._client().delete_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=s3_key,
|
||||
)
|
||||
|
||||
async def list_keys(self, user_id: str, table: str) -> list[str]:
|
||||
"""Return all S3 keys for a given user + table combination.
|
||||
|
||||
Uses the prefix ``{user_id}/{table}/`` to scope the listing.
|
||||
"""
|
||||
prefix = f"{user_id}/{table}/"
|
||||
response = self._client().list_objects_v2(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Prefix=prefix,
|
||||
)
|
||||
return [obj["Key"] for obj in response.get("Contents", [])]
|
||||
@@ -1,32 +0,0 @@
|
||||
"""Integrity verification only — the backend NEVER decrypts user data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
def verify_checksum(blob: bytes, checksum: str) -> bool:
|
||||
"""Return ``True`` if SHA-256(blob) matches *checksum*.
|
||||
|
||||
Uses ``hmac.compare_digest`` for constant-time comparison to prevent
|
||||
timing-based side-channel attacks.
|
||||
"""
|
||||
computed = hashlib.sha256(blob).hexdigest()
|
||||
return hmac.compare_digest(computed, checksum)
|
||||
|
||||
|
||||
def reject_if_tampered(blob: bytes, checksum: str) -> None:
|
||||
"""Raise ``HTTP 400`` if the blob does not match its checksum.
|
||||
|
||||
Call this before storing or forwarding any client-provided blob.
|
||||
The backend never holds decryption keys — this check only verifies
|
||||
that the opaque bytes arrived intact.
|
||||
"""
|
||||
if not verify_checksum(blob, checksum):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Checksum mismatch: blob integrity check failed",
|
||||
)
|
||||
@@ -1,205 +0,0 @@
|
||||
"""Cloud vector store — wraps Pinecone (default) or Qdrant.
|
||||
|
||||
Vectors are pre-encrypted blobs from the client. The backend stores them
|
||||
alongside a deterministic 32-dim float representation derived from the blob's
|
||||
SHA-256 hash. Semantic ANN search is not meaningful on encrypted data — this
|
||||
is a known trade-off documented in the backend plan.
|
||||
|
||||
Isolation: Pinecone uses ``namespace=user_id``; Qdrant filters by
|
||||
``user_id`` payload field on a shared collection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
from pinecone import Pinecone
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue, PointIdsList, PointStruct
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.schemas import VectorItem, VectorSearchResult
|
||||
|
||||
_QDRANT_COLLECTION = "adiuva_vectors"
|
||||
|
||||
|
||||
def _blob_to_vector(blob: bytes) -> list[float]:
|
||||
"""Derive a 32-dim float vector from *blob* for storage purposes only.
|
||||
|
||||
Uses SHA-256 to produce a deterministic 32-byte fingerprint, then
|
||||
normalises each byte to the range [-1.0, 1.0]. This vector carries no
|
||||
semantic meaning on encrypted data.
|
||||
"""
|
||||
return [(b - 128) / 128.0 for b in hashlib.sha256(blob).digest()]
|
||||
|
||||
|
||||
class VectorStore:
|
||||
"""Thin wrapper around Pinecone or Qdrant.
|
||||
|
||||
The backend to use is selected at runtime:
|
||||
- Pinecone: when ``settings.PINECONE_API_KEY`` is non-empty.
|
||||
- Qdrant: otherwise (requires ``settings.QDRANT_URL``).
|
||||
"""
|
||||
|
||||
def _use_pinecone(self) -> bool:
|
||||
return bool(settings.PINECONE_API_KEY)
|
||||
|
||||
# ── Pinecone helpers ──────────────────────────────────────────────
|
||||
|
||||
def _pinecone_index(self) -> Any:
|
||||
pc = Pinecone(api_key=settings.PINECONE_API_KEY)
|
||||
return pc.Index(settings.PINECONE_INDEX)
|
||||
|
||||
# ── Qdrant helpers ────────────────────────────────────────────────
|
||||
|
||||
def _qdrant_client(self) -> Any:
|
||||
return QdrantClient(
|
||||
url=settings.QDRANT_URL,
|
||||
api_key=settings.QDRANT_API_KEY or None,
|
||||
)
|
||||
|
||||
# ── Public API ────────────────────────────────────────────────────
|
||||
|
||||
async def upsert(self, user_id: str, vectors: list[VectorItem]) -> None:
|
||||
"""Store encrypted vectors in the backend.
|
||||
|
||||
Each ``VectorItem.blob`` is base64-encoded and kept in metadata/payload
|
||||
so it can be returned verbatim during search.
|
||||
|
||||
Args:
|
||||
user_id: Used as Pinecone namespace or Qdrant payload field.
|
||||
vectors: List of encrypted vector items from the client.
|
||||
"""
|
||||
if self._use_pinecone():
|
||||
await self._pinecone_upsert(user_id, vectors)
|
||||
else:
|
||||
await self._qdrant_upsert(user_id, vectors)
|
||||
|
||||
async def search(
|
||||
self,
|
||||
user_id: str,
|
||||
query_blob: bytes,
|
||||
top_k: int,
|
||||
) -> list[VectorSearchResult]:
|
||||
"""Query the vector store and return encrypted result blobs.
|
||||
|
||||
The query vector is derived from *query_blob* using the same
|
||||
deterministic mapping as upsert.
|
||||
|
||||
Args:
|
||||
user_id: Scopes the search to this user's namespace.
|
||||
query_blob: Encrypted query from the client.
|
||||
top_k: Maximum number of results to return.
|
||||
|
||||
Returns:
|
||||
List of ``VectorSearchResult`` with ``id``, ``score``, and ``blob``.
|
||||
"""
|
||||
if self._use_pinecone():
|
||||
return await self._pinecone_search(user_id, query_blob, top_k)
|
||||
return await self._qdrant_search(user_id, query_blob, top_k)
|
||||
|
||||
async def delete(self, user_id: str, vector_ids: list[str]) -> None:
|
||||
"""Remove vectors by ID, scoped to *user_id*.
|
||||
|
||||
Args:
|
||||
user_id: Namespace / payload filter to prevent cross-user deletion.
|
||||
vector_ids: List of vector IDs to remove.
|
||||
"""
|
||||
if self._use_pinecone():
|
||||
await self._pinecone_delete(user_id, vector_ids)
|
||||
else:
|
||||
await self._qdrant_delete(user_id, vector_ids)
|
||||
|
||||
# ── Pinecone implementation ───────────────────────────────────────
|
||||
|
||||
async def _pinecone_upsert(self, user_id: str, vectors: list[VectorItem]) -> None:
|
||||
index = self._pinecone_index()
|
||||
records = [
|
||||
{
|
||||
"id": v.id,
|
||||
"values": _blob_to_vector(v.blob),
|
||||
"metadata": {
|
||||
"blob": base64.b64encode(v.blob).decode(),
|
||||
"checksum": v.checksum,
|
||||
"user_id": user_id,
|
||||
},
|
||||
}
|
||||
for v in vectors
|
||||
]
|
||||
index.upsert(vectors=records, namespace=user_id)
|
||||
|
||||
async def _pinecone_search(
|
||||
self, user_id: str, query_blob: bytes, top_k: int
|
||||
) -> list[VectorSearchResult]:
|
||||
index = self._pinecone_index()
|
||||
query_vector = _blob_to_vector(query_blob)
|
||||
response = index.query(
|
||||
vector=query_vector,
|
||||
top_k=top_k,
|
||||
namespace=user_id,
|
||||
include_metadata=True,
|
||||
)
|
||||
results: list[VectorSearchResult] = []
|
||||
for match in response.get("matches", []):
|
||||
blob_bytes = base64.b64decode(match["metadata"]["blob"])
|
||||
results.append(
|
||||
VectorSearchResult(
|
||||
id=match["id"],
|
||||
score=match["score"],
|
||||
blob=blob_bytes,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
async def _pinecone_delete(self, user_id: str, vector_ids: list[str]) -> None:
|
||||
index = self._pinecone_index()
|
||||
index.delete(ids=vector_ids, namespace=user_id)
|
||||
|
||||
# ── Qdrant implementation ─────────────────────────────────────────
|
||||
|
||||
async def _qdrant_upsert(self, user_id: str, vectors: list[VectorItem]) -> None:
|
||||
client = self._qdrant_client()
|
||||
points = [
|
||||
PointStruct(
|
||||
id=v.id,
|
||||
vector=_blob_to_vector(v.blob),
|
||||
payload={
|
||||
"blob": base64.b64encode(v.blob).decode(),
|
||||
"checksum": v.checksum,
|
||||
"user_id": user_id,
|
||||
},
|
||||
)
|
||||
for v in vectors
|
||||
]
|
||||
client.upsert(collection_name=_QDRANT_COLLECTION, points=points)
|
||||
|
||||
async def _qdrant_search(
|
||||
self, user_id: str, query_blob: bytes, top_k: int
|
||||
) -> list[VectorSearchResult]:
|
||||
client = self._qdrant_client()
|
||||
query_vector = _blob_to_vector(query_blob)
|
||||
hits = client.search(
|
||||
collection_name=_QDRANT_COLLECTION,
|
||||
query_vector=query_vector,
|
||||
query_filter=Filter(
|
||||
must=[FieldCondition(key="user_id", match=MatchValue(value=user_id))]
|
||||
),
|
||||
limit=top_k,
|
||||
)
|
||||
return [
|
||||
VectorSearchResult(
|
||||
id=str(hit.id),
|
||||
score=hit.score,
|
||||
blob=base64.b64decode(hit.payload["blob"]),
|
||||
)
|
||||
for hit in hits
|
||||
]
|
||||
|
||||
async def _qdrant_delete(self, user_id: str, vector_ids: list[str]) -> None:
|
||||
client = self._qdrant_client()
|
||||
client.delete(
|
||||
collection_name=_QDRANT_COLLECTION,
|
||||
points_selector=PointIdsList(points=vector_ids),
|
||||
)
|
||||
Reference in New Issue
Block a user