Refactor tests for execution plan and add comprehensive storage tests
- Updated `TestModuleSingletons` in `test_execution_plan.py` to reflect new agent templates and playbook names. - Changed assertions in playbook tests to match updated templates and agents. - Introduced `test_storage.py` to cover the storage layer, including encryption, BlobStore, and VectorStore functionalities. - Added tests for S3 interactions, ensuring upload, download, delete, and list operations work as expected. - Implemented mock tests for Pinecone and Qdrant vector stores to validate upsert, search, and delete operations.
This commit is contained in:
105
app/storage/blob_store.py
Normal file
105
app/storage/blob_store.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""S3-backed store for E2E-encrypted blobs.
|
||||
|
||||
Keys are structured as ``{user_id}/{table}/{record_id}``.
|
||||
The backend never inspects blob content — it stores and retrieves opaque bytes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from app.config.settings import settings
|
||||
|
||||
|
||||
class BlobStore:
|
||||
"""Thin wrapper around boto3 S3.
|
||||
|
||||
All blobs must be E2E encrypted by the client before upload.
|
||||
The backend adds SSE-S3 as an extra layer of at-rest encryption
|
||||
but cannot decrypt the inner client-side payload.
|
||||
"""
|
||||
|
||||
def _client(self) -> Any:
|
||||
return boto3.client(
|
||||
"s3",
|
||||
region_name=settings.S3_REGION,
|
||||
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _key(user_id: str, table: str, record_id: str) -> str:
|
||||
return f"{user_id}/{table}/{record_id}"
|
||||
|
||||
async def upload(
|
||||
self,
|
||||
user_id: str,
|
||||
table: str,
|
||||
record_id: str,
|
||||
blob: bytes,
|
||||
checksum: str,
|
||||
) -> str:
|
||||
"""Store *blob* in S3 and return the S3 key.
|
||||
|
||||
Args:
|
||||
user_id: Owner of the blob (used as key prefix).
|
||||
table: Logical table name (e.g. ``"tasks"``).
|
||||
record_id: Record UUID.
|
||||
blob: Raw bytes (pre-encrypted by client).
|
||||
checksum: SHA-256 hex digest supplied by the client; stored as
|
||||
object metadata for download-time verification.
|
||||
|
||||
Returns:
|
||||
The S3 key under which the blob was stored.
|
||||
"""
|
||||
key = self._key(user_id, table, record_id)
|
||||
self._client().put_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=key,
|
||||
Body=blob,
|
||||
ServerSideEncryption="AES256", # SSE-S3 at rest
|
||||
Metadata={"checksum": checksum},
|
||||
)
|
||||
return key
|
||||
|
||||
async def download(self, user_id: str, s3_key: str) -> bytes:
|
||||
"""Retrieve the blob stored at *s3_key*.
|
||||
|
||||
*user_id* is retained in the signature so higher-level code can
|
||||
enforce ownership without re-parsing the key.
|
||||
|
||||
Raises:
|
||||
``botocore.exceptions.ClientError`` with code ``NoSuchKey`` if the
|
||||
object does not exist.
|
||||
"""
|
||||
response = self._client().get_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=s3_key,
|
||||
)
|
||||
return response["Body"].read()
|
||||
|
||||
async def delete(self, user_id: str, s3_key: str) -> None:
|
||||
"""Delete the object at *s3_key*.
|
||||
|
||||
S3 ``delete_object`` is idempotent — it succeeds even if the key does
|
||||
not exist.
|
||||
"""
|
||||
self._client().delete_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=s3_key,
|
||||
)
|
||||
|
||||
async def list_keys(self, user_id: str, table: str) -> list[str]:
|
||||
"""Return all S3 keys for a given user + table combination.
|
||||
|
||||
Uses the prefix ``{user_id}/{table}/`` to scope the listing.
|
||||
"""
|
||||
prefix = f"{user_id}/{table}/"
|
||||
response = self._client().list_objects_v2(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Prefix=prefix,
|
||||
)
|
||||
return [obj["Key"] for obj in response.get("Contents", [])]
|
||||
Reference in New Issue
Block a user