feat(step-3.6): cloud provider integrations (Gmail, Outlook, Teams)
- Add app/integrations/__init__.py: Fernet token encryption helpers, EmailMessage/ChatMessage dataclasses, get_provider() factory - Add app/integrations/gmail.py: GmailClient with async fetch_messages(), token refresh, configurable label/sender/date filters - Add app/integrations/ms_graph.py: MSGraphClient with fetch_emails() (Outlook) and fetch_messages() (Teams), MSAL token refresh, OData filters - Update app/core/agent_runner.py: replace run_cloud_agent() stub with full 8-step implementation; extend _finalize_run() for cloud config type - Update app/config/settings.py: add OAuth + Fernet encryption settings - Update requirements.txt: google-api-python-client, google-auth-*, msal, cryptography - Add tests/test_integrations.py: 47 tests covering all integration code - Update tests/test_agent_runner.py: replace stub test with 7 real tests All 76 new/updated tests pass.
This commit is contained in:
164
app/integrations/__init__.py
Normal file
164
app/integrations/__init__.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""Cloud provider integration utilities.
|
||||
|
||||
Provides:
|
||||
* Shared message dataclasses (``EmailMessage``, ``ChatMessage``) used by
|
||||
both the Gmail and MS Graph clients and consumed by ``agent_runner``.
|
||||
* ``get_provider()`` — factory that returns the correct client given a
|
||||
provider name and decrypted OAuth credentials dict.
|
||||
* ``encrypt_token()`` / ``decrypt_token()`` — Fernet-based at-rest
|
||||
encryption for OAuth tokens stored in ``cloud_agent_configs``.
|
||||
|
||||
Encryption rationale
|
||||
--------------------
|
||||
Unlike user content (which is E2E-encrypted client-side and **never**
|
||||
decrypted server-side), OAuth tokens *must* be decrypted server-side
|
||||
because the backend makes provider API calls on behalf of the user.
|
||||
The Fernet key lives solely in ``OAUTH_ENCRYPTION_KEY`` env var — it
|
||||
is never returned to clients.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from app.config.settings import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.integrations.gmail import GmailClient
|
||||
from app.integrations.ms_graph import MSGraphClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Shared message types ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmailMessage:
|
||||
"""A single email message fetched from Gmail or Outlook."""
|
||||
|
||||
id: str
|
||||
subject: str
|
||||
sender: str
|
||||
body_text: str
|
||||
date: datetime
|
||||
labels: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def as_text(self) -> str:
|
||||
"""Return a human-readable text representation for LLM extraction."""
|
||||
date_str = self.date.strftime("%Y-%m-%d %H:%M")
|
||||
labels_str = f" [{', '.join(self.labels)}]" if self.labels else ""
|
||||
return (
|
||||
f"From: {self.sender}\n"
|
||||
f"Date: {date_str}{labels_str}\n"
|
||||
f"Subject: {self.subject}\n\n"
|
||||
f"{self.body_text}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatMessage:
|
||||
"""A single Teams chat or channel message fetched from MS Graph."""
|
||||
|
||||
id: str
|
||||
content: str
|
||||
sender: str
|
||||
channel: str | None
|
||||
date: datetime
|
||||
|
||||
@property
|
||||
def as_text(self) -> str:
|
||||
"""Return a human-readable text representation for LLM extraction."""
|
||||
date_str = self.date.strftime("%Y-%m-%d %H:%M")
|
||||
channel_str = f" [channel: {self.channel}]" if self.channel else ""
|
||||
return (
|
||||
f"From: {self.sender}\n"
|
||||
f"Date: {date_str}{channel_str}\n\n"
|
||||
f"{self.content}"
|
||||
)
|
||||
|
||||
|
||||
# ── Fernet helpers ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _get_fernet() -> Fernet:
|
||||
"""Return a ``Fernet`` instance using ``settings.OAUTH_ENCRYPTION_KEY``.
|
||||
|
||||
Raises ``RuntimeError`` if ``OAUTH_ENCRYPTION_KEY`` is not set — callers
|
||||
must ensure this is configured before persisting OAuth tokens.
|
||||
"""
|
||||
key = settings.OAUTH_ENCRYPTION_KEY
|
||||
if not key:
|
||||
raise RuntimeError(
|
||||
"OAUTH_ENCRYPTION_KEY is not set. "
|
||||
"Generate one with: python -c \"from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())\""
|
||||
)
|
||||
return Fernet(key.encode() if isinstance(key, str) else key)
|
||||
|
||||
|
||||
def encrypt_token(token_info: dict) -> str:
|
||||
"""Fernet-encrypt an OAuth credential dict and return a base64 string.
|
||||
|
||||
Stores the full ``{access_token, refresh_token, token_uri, client_id,
|
||||
client_secret, scopes, expiry}`` dict (or equivalent MSAL shape).
|
||||
|
||||
Raises:
|
||||
RuntimeError: OAUTH_ENCRYPTION_KEY is not configured.
|
||||
ValueError: ``token_info`` is not a non-empty dict.
|
||||
"""
|
||||
if not isinstance(token_info, dict) or not token_info:
|
||||
raise ValueError("token_info must be a non-empty dict")
|
||||
plaintext = json.dumps(token_info).encode("utf-8")
|
||||
return _get_fernet().encrypt(plaintext).decode("utf-8")
|
||||
|
||||
|
||||
def decrypt_token(encrypted: str) -> dict:
|
||||
"""Decrypt a Fernet-encrypted token string and return the credential dict.
|
||||
|
||||
Raises:
|
||||
RuntimeError: OAUTH_ENCRYPTION_KEY is not configured.
|
||||
ValueError: The encrypted string is invalid or was encrypted with a
|
||||
different key.
|
||||
"""
|
||||
try:
|
||||
plaintext = _get_fernet().decrypt(encrypted.encode("utf-8"))
|
||||
return json.loads(plaintext)
|
||||
except (InvalidToken, json.JSONDecodeError) as exc:
|
||||
raise ValueError(f"Failed to decrypt OAuth token: {exc}") from exc
|
||||
|
||||
|
||||
# ── Provider factory ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def get_provider(
|
||||
provider: str,
|
||||
credentials_info: dict,
|
||||
) -> "GmailClient | MSGraphClient":
|
||||
"""Return the correct provider client for *provider*.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
provider:
|
||||
One of ``"gmail"``, ``"outlook"``, ``"teams"``.
|
||||
credentials_info:
|
||||
Decrypted OAuth credential dict (Google or Microsoft shape).
|
||||
|
||||
Raises:
|
||||
ValueError: Unknown provider name.
|
||||
"""
|
||||
if provider == "gmail":
|
||||
from app.integrations.gmail import GmailClient
|
||||
return GmailClient(credentials_info)
|
||||
if provider in {"outlook", "teams"}:
|
||||
from app.integrations.ms_graph import MSGraphClient
|
||||
return MSGraphClient(credentials_info)
|
||||
raise ValueError(
|
||||
f"Unknown cloud provider {provider!r}. "
|
||||
"Supported: 'gmail', 'outlook', 'teams'."
|
||||
)
|
||||
335
app/integrations/gmail.py
Normal file
335
app/integrations/gmail.py
Normal file
@@ -0,0 +1,335 @@
|
||||
"""Gmail API client for cloud agent integration.
|
||||
|
||||
Wraps the Google Gmail REST API to fetch email messages matching a
|
||||
``filter_config`` dict. Uses the official ``google-api-python-client``
|
||||
library (synchronous) wrapped in ``asyncio.to_thread()`` to avoid
|
||||
blocking the event loop.
|
||||
|
||||
Token refresh is handled transparently: when the stored access token has
|
||||
expired, ``google.auth.transport.requests.Request`` will use the refresh
|
||||
token to obtain a fresh one. The caller is responsible for persisting
|
||||
any refreshed credentials back to ``CloudAgentConfig.oauth_token_encrypted``
|
||||
(see ``agent_runner.run_cloud_agent``).
|
||||
|
||||
Credential dict shape (Google OAuth2):
|
||||
{
|
||||
"token": "<access_token>",
|
||||
"refresh_token": "<refresh_token>",
|
||||
"token_uri": "https://oauth2.googleapis.com/token",
|
||||
"client_id": "<client_id>",
|
||||
"client_secret": "<client_secret>",
|
||||
"scopes": ["https://www.googleapis.com/auth/gmail.readonly"],
|
||||
"expiry": "2025-01-01T00:00:00Z" # optional ISO-8601
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import email
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.integrations import EmailMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Gmail search date format — e.g. "after:2025/01/01"
|
||||
_GMAIL_DATE_FMT = "%Y/%m/%d"
|
||||
|
||||
# Maximum characters of body text forwarded to the LLM.
|
||||
_BODY_TRUNCATE = 8_000
|
||||
|
||||
# Maximum messages retrieved per run (prevents runaway quota usage).
|
||||
_MAX_MESSAGES = 200
|
||||
|
||||
|
||||
def _build_gmail_query(
|
||||
filter_config: dict[str, Any] | None,
|
||||
since: datetime | None,
|
||||
) -> str:
|
||||
"""Build a Gmail search query string from *filter_config* and *since*.
|
||||
|
||||
Supported ``filter_config`` keys:
|
||||
labels (list[str]): Gmail label names, e.g. ``["INBOX", "work"]``
|
||||
senders (list[str]): Sender addresses or domains to include
|
||||
date_range (dict): ``{from: "<YYYY-MM-DD>", to: "<YYYY-MM-DD>"}``
|
||||
|
||||
A hard ``since`` date (from last run) always overrides ``date_range.from``
|
||||
when it is earlier.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
cfg = filter_config or {}
|
||||
|
||||
# Labels — joined with OR when multiple given.
|
||||
labels: list[str] = cfg.get("labels", [])
|
||||
if labels:
|
||||
if len(labels) == 1:
|
||||
parts.append(f"label:{labels[0]}")
|
||||
else:
|
||||
label_expr = " OR ".join(f"label:{lbl}" for lbl in labels)
|
||||
parts.append(f"({label_expr})")
|
||||
|
||||
# Senders — each prefixed with "from:".
|
||||
senders: list[str] = cfg.get("senders", [])
|
||||
for sender in senders:
|
||||
parts.append(f"from:{sender}")
|
||||
|
||||
# Date range.
|
||||
date_range: dict = cfg.get("date_range", {})
|
||||
from_str: str | None = date_range.get("from")
|
||||
to_str: str | None = date_range.get("to")
|
||||
|
||||
# Determine effective "from" date: most recent of filter_config.date_range.from and since.
|
||||
effective_since: datetime | None = since
|
||||
if from_str:
|
||||
try:
|
||||
cfg_since = datetime.fromisoformat(from_str.replace("Z", "+00:00"))
|
||||
if cfg_since.tzinfo is None:
|
||||
cfg_since = cfg_since.replace(tzinfo=timezone.utc)
|
||||
if effective_since is None or cfg_since > effective_since:
|
||||
effective_since = cfg_since
|
||||
except ValueError:
|
||||
logger.warning("gmail: invalid date_range.from %r — ignoring", from_str)
|
||||
|
||||
if effective_since:
|
||||
parts.append(f"after:{effective_since.strftime(_GMAIL_DATE_FMT)}")
|
||||
|
||||
if to_str:
|
||||
try:
|
||||
to_dt = datetime.fromisoformat(to_str.replace("Z", "+00:00"))
|
||||
parts.append(f"before:{to_dt.strftime(_GMAIL_DATE_FMT)}")
|
||||
except ValueError:
|
||||
logger.warning("gmail: invalid date_range.to %r — ignoring", to_str)
|
||||
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _strip_html(raw_html: str) -> str:
|
||||
"""Remove HTML tags and decode entities to get plain text."""
|
||||
no_tags = re.sub(r"<[^>]+>", " ", raw_html)
|
||||
decoded = html.unescape(no_tags)
|
||||
return re.sub(r"\s+", " ", decoded).strip()
|
||||
|
||||
|
||||
def _parse_body(payload: dict[str, Any]) -> str:
|
||||
"""Recursively extract the plain-text body from a Gmail message payload.
|
||||
|
||||
Prefers ``text/plain``; falls back to ``text/html`` (stripped of tags).
|
||||
Returns an empty string if no body can be extracted.
|
||||
"""
|
||||
mime_type: str = payload.get("mimeType", "")
|
||||
body: dict = payload.get("body", {})
|
||||
parts: list[dict] = payload.get("parts", [])
|
||||
|
||||
if mime_type == "text/plain":
|
||||
data = body.get("data", "")
|
||||
if data:
|
||||
return base64.urlsafe_b64decode(data + "==").decode("utf-8", errors="replace")
|
||||
return ""
|
||||
|
||||
if mime_type == "text/html":
|
||||
data = body.get("data", "")
|
||||
if data:
|
||||
raw = base64.urlsafe_b64decode(data + "==").decode("utf-8", errors="replace")
|
||||
return _strip_html(raw)
|
||||
return ""
|
||||
|
||||
# Multipart — prefer text/plain part, fall back to text/html.
|
||||
plain_fallback = ""
|
||||
for part in parts:
|
||||
part_mime = part.get("mimeType", "")
|
||||
if part_mime == "text/plain":
|
||||
return _parse_body(part)
|
||||
if part_mime == "text/html" and not plain_fallback:
|
||||
plain_fallback = _parse_body(part)
|
||||
if part_mime.startswith("multipart/"):
|
||||
nested = _parse_body(part)
|
||||
if nested:
|
||||
return nested
|
||||
return plain_fallback
|
||||
|
||||
|
||||
def _parse_date(raw: str) -> datetime:
|
||||
"""Parse an RFC 2822 email date header into a UTC ``datetime``."""
|
||||
try:
|
||||
parsed = email.utils.parsedate_to_datetime(raw)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
except Exception:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class GmailClient:
|
||||
"""Fetch email messages from a Gmail account via the Gmail REST API.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
credentials_info:
|
||||
Decrypted OAuth2 credential dict. Must contain at minimum
|
||||
``token`` (access token) or ``refresh_token`` + ``token_uri`` +
|
||||
``client_id`` + ``client_secret``.
|
||||
"""
|
||||
|
||||
def __init__(self, credentials_info: dict[str, Any]) -> None:
|
||||
from google.oauth2.credentials import Credentials
|
||||
|
||||
self._credentials_info = credentials_info
|
||||
expiry_str: str | None = credentials_info.get("expiry")
|
||||
expiry: datetime | None = None
|
||||
if expiry_str:
|
||||
try:
|
||||
expiry = datetime.fromisoformat(
|
||||
expiry_str.replace("Z", "+00:00")
|
||||
).replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
self._credentials = Credentials(
|
||||
token=credentials_info.get("token"),
|
||||
refresh_token=credentials_info.get("refresh_token"),
|
||||
token_uri=credentials_info.get("token_uri", "https://oauth2.googleapis.com/token"),
|
||||
client_id=credentials_info.get("client_id"),
|
||||
client_secret=credentials_info.get("client_secret"),
|
||||
scopes=credentials_info.get("scopes"),
|
||||
expiry=expiry,
|
||||
)
|
||||
|
||||
# ── Public API ─────────────────────────────────────────────────────────
|
||||
|
||||
async def fetch_messages(
|
||||
self,
|
||||
filter_config: dict[str, Any] | None = None,
|
||||
since: datetime | None = None,
|
||||
) -> list[EmailMessage]:
|
||||
"""Return up to ``_MAX_MESSAGES`` emails matching *filter_config*.
|
||||
|
||||
Runs the synchronous Google API calls inside ``asyncio.to_thread()``
|
||||
to avoid blocking the async event loop.
|
||||
|
||||
Token refresh is performed automatically when the access token has
|
||||
expired. After the call, ``self.refreshed_credentials`` may be
|
||||
consulted to detect whether new credentials should be persisted.
|
||||
"""
|
||||
query = _build_gmail_query(filter_config, since)
|
||||
logger.debug("gmail: executing search query %r", query)
|
||||
return await asyncio.to_thread(self._fetch_sync, query)
|
||||
|
||||
@property
|
||||
def refreshed_credentials(self) -> dict[str, Any] | None:
|
||||
"""Return updated credential dict if the access token was refreshed.
|
||||
|
||||
If the credentials were refreshed during ``fetch_messages()``, returns
|
||||
a new dict that should be re-encrypted and written back to the DB.
|
||||
Returns ``None`` if no refresh occurred.
|
||||
"""
|
||||
creds = self._credentials
|
||||
if not creds.valid and creds.expired:
|
||||
return None
|
||||
# Check whether the token changed from what was stored.
|
||||
if creds.token != self._credentials_info.get("token"):
|
||||
result = {
|
||||
"token": creds.token,
|
||||
"refresh_token": creds.refresh_token,
|
||||
"token_uri": creds.token_uri,
|
||||
"client_id": creds.client_id,
|
||||
"client_secret": creds.client_secret,
|
||||
"scopes": list(creds.scopes or []),
|
||||
}
|
||||
if creds.expiry:
|
||||
result["expiry"] = creds.expiry.isoformat()
|
||||
return result
|
||||
return None
|
||||
|
||||
# ── Internal sync worker ───────────────────────────────────────────────
|
||||
|
||||
def _fetch_sync(self, query: str) -> list[EmailMessage]:
|
||||
"""Synchronous worker — called inside ``asyncio.to_thread()``."""
|
||||
import googleapiclient.discovery
|
||||
import googleapiclient.errors
|
||||
from google.auth.transport.requests import Request
|
||||
|
||||
# Refresh token if needed before building the service.
|
||||
if self._credentials.expired and self._credentials.refresh_token:
|
||||
try:
|
||||
self._credentials.refresh(Request())
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"Gmail token refresh failed: {exc}") from exc
|
||||
|
||||
service = googleapiclient.discovery.build(
|
||||
"gmail", "v1", credentials=self._credentials, cache_discovery=False
|
||||
)
|
||||
user_api = service.users() # type: ignore[attr-defined]
|
||||
|
||||
# ── List matching message IDs ──────────────────────────────────────
|
||||
ids: list[str] = []
|
||||
page_token: str | None = None
|
||||
while len(ids) < _MAX_MESSAGES:
|
||||
batch_size = min(100, _MAX_MESSAGES - len(ids))
|
||||
kwargs: dict[str, Any] = {
|
||||
"userId": "me",
|
||||
"maxResults": batch_size,
|
||||
}
|
||||
if query:
|
||||
kwargs["q"] = query
|
||||
if page_token:
|
||||
kwargs["pageToken"] = page_token
|
||||
|
||||
try:
|
||||
resp = user_api.messages().list(**kwargs).execute()
|
||||
except googleapiclient.errors.HttpError as exc:
|
||||
raise RuntimeError(f"Gmail messages.list failed: {exc}") from exc
|
||||
|
||||
for msg in resp.get("messages", []):
|
||||
ids.append(msg["id"])
|
||||
|
||||
page_token = resp.get("nextPageToken")
|
||||
if not page_token:
|
||||
break
|
||||
|
||||
if not ids:
|
||||
logger.debug("gmail: no messages matched query %r", query)
|
||||
return []
|
||||
|
||||
logger.info("gmail: fetching %d message(s)", len(ids))
|
||||
|
||||
# ── Fetch individual message details ──────────────────────────────
|
||||
messages: list[EmailMessage] = []
|
||||
for msg_id in ids:
|
||||
try:
|
||||
msg = user_api.messages().get(
|
||||
userId="me", id=msg_id, format="full"
|
||||
).execute()
|
||||
|
||||
headers: dict[str, str] = {
|
||||
h["name"].lower(): h["value"]
|
||||
for h in msg.get("payload", {}).get("headers", [])
|
||||
}
|
||||
subject = headers.get("subject", "(no subject)")
|
||||
sender = headers.get("from", "unknown")
|
||||
date_raw = headers.get("date", "")
|
||||
date = _parse_date(date_raw) if date_raw else datetime.now(timezone.utc)
|
||||
|
||||
body_text = _parse_body(msg.get("payload", {}))[:_BODY_TRUNCATE]
|
||||
labels = msg.get("labelIds", [])
|
||||
|
||||
messages.append(EmailMessage(
|
||||
id=msg_id,
|
||||
subject=subject,
|
||||
sender=sender,
|
||||
body_text=body_text,
|
||||
date=date,
|
||||
labels=labels,
|
||||
))
|
||||
except googleapiclient.errors.HttpError as exc:
|
||||
logger.warning("gmail: skipping message %s — HTTP error: %s", msg_id, exc)
|
||||
except Exception as exc:
|
||||
logger.warning("gmail: skipping message %s — unexpected error: %s", msg_id, exc)
|
||||
|
||||
logger.info("gmail: returned %d message(s)", len(messages))
|
||||
return messages
|
||||
352
app/integrations/ms_graph.py
Normal file
352
app/integrations/ms_graph.py
Normal file
@@ -0,0 +1,352 @@
|
||||
"""Microsoft Graph API client for Outlook and Teams cloud agent integration.
|
||||
|
||||
Handles two data sources:
|
||||
|
||||
* **Outlook email** (``provider="outlook"``) — ``fetch_emails()`` calls
|
||||
``/me/messages`` with an OData ``$filter`` built from ``filter_config``.
|
||||
* **Teams messages** (``provider="teams"``) — ``fetch_messages()`` calls
|
||||
``/me/chats/getAllMessages`` filtered by date.
|
||||
|
||||
Authentication uses MSAL ``PublicClientApplication`` to acquire a token
|
||||
from a stored refresh token. The ``httpx.AsyncClient`` (already a project
|
||||
dependency) is used for all API calls.
|
||||
|
||||
Credential dict shape (Microsoft OAuth2 / MSAL):
|
||||
{
|
||||
"access_token": "<access_token>",
|
||||
"refresh_token": "<refresh_token>",
|
||||
"token_type": "Bearer",
|
||||
"scope": "Mail.Read ChannelMessage.Read.All offline_access",
|
||||
"expires_in": 3600
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.integrations import ChatMessage, EmailMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_GRAPH_BASE = "https://graph.microsoft.com/v1.0"
|
||||
|
||||
# Max items fetched per run.
|
||||
_MAX_EMAILS = 200
|
||||
_MAX_MESSAGES = 200
|
||||
|
||||
# Max characters of body forwarded to the LLM.
|
||||
_BODY_TRUNCATE = 8_000
|
||||
|
||||
|
||||
def _strip_html(raw: str) -> str:
|
||||
"""Strip HTML tags and collapse whitespace."""
|
||||
no_tags = re.sub(r"<[^>]+>", " ", raw)
|
||||
import html as _html
|
||||
decoded = _html.unescape(no_tags)
|
||||
return re.sub(r"\s+", " ", decoded).strip()
|
||||
|
||||
|
||||
def _odata_datetime(dt: datetime) -> str:
|
||||
"""Format a datetime as an OData datetime literal (UTC, ISO 8601)."""
|
||||
utc = dt.astimezone(timezone.utc)
|
||||
return utc.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _build_email_filter(
|
||||
filter_config: dict[str, Any] | None,
|
||||
since: datetime | None,
|
||||
) -> str:
|
||||
"""Build an OData ``$filter`` expression for the ``/me/messages`` endpoint.
|
||||
|
||||
Supported ``filter_config`` keys:
|
||||
senders (list[str]): Sender email addresses.
|
||||
date_range (dict): ``{from: "<ISO-8601>", to: "<ISO-8601>"}``
|
||||
folders (list[str]): Folder display names (not directly filterable
|
||||
via OData, so ignored here — callers iterate
|
||||
folder IDs separately if needed; listed for
|
||||
completeness).
|
||||
|
||||
A hard ``since`` date always overrides ``date_range.from`` when it is
|
||||
earlier.
|
||||
"""
|
||||
clauses: list[str] = []
|
||||
cfg = filter_config or {}
|
||||
|
||||
# Senders.
|
||||
senders: list[str] = cfg.get("senders", [])
|
||||
if senders:
|
||||
sender_clauses = [f"from/emailAddress/address eq '{s}'" for s in senders]
|
||||
clauses.append("(" + " or ".join(sender_clauses) + ")")
|
||||
|
||||
# Date range.
|
||||
date_range: dict = cfg.get("date_range", {})
|
||||
from_str: str | None = date_range.get("from")
|
||||
|
||||
effective_since: datetime | None = since
|
||||
if from_str:
|
||||
try:
|
||||
cfg_since = datetime.fromisoformat(from_str.replace("Z", "+00:00"))
|
||||
if cfg_since.tzinfo is None:
|
||||
cfg_since = cfg_since.replace(tzinfo=timezone.utc)
|
||||
if effective_since is None or cfg_since > effective_since:
|
||||
effective_since = cfg_since
|
||||
except ValueError:
|
||||
logger.warning("ms_graph: invalid date_range.from %r — ignoring", from_str)
|
||||
|
||||
if effective_since:
|
||||
clauses.append(f"receivedDateTime ge {_odata_datetime(effective_since)}")
|
||||
|
||||
to_str: str | None = date_range.get("to")
|
||||
if to_str:
|
||||
try:
|
||||
to_dt = datetime.fromisoformat(to_str.replace("Z", "+00:00"))
|
||||
if to_dt.tzinfo is None:
|
||||
to_dt = to_dt.replace(tzinfo=timezone.utc)
|
||||
clauses.append(f"receivedDateTime le {_odata_datetime(to_dt)}")
|
||||
except ValueError:
|
||||
logger.warning("ms_graph: invalid date_range.to %r — ignoring", to_str)
|
||||
|
||||
return " and ".join(clauses)
|
||||
|
||||
|
||||
class MSGraphClient:
|
||||
"""Fetch emails and Teams messages via the Microsoft Graph REST API.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
credentials_info:
|
||||
Decrypted MSAL credential dict.
|
||||
"""
|
||||
|
||||
def __init__(self, credentials_info: dict[str, Any]) -> None:
|
||||
self._credentials_info = credentials_info
|
||||
self._access_token: str = credentials_info.get("access_token", "")
|
||||
self._original_access_token: str = self._access_token
|
||||
self._refresh_token: str | None = credentials_info.get("refresh_token")
|
||||
|
||||
# ── Token management ───────────────────────────────────────────────────
|
||||
|
||||
def _auth_headers(self) -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {self._access_token}"}
|
||||
|
||||
async def _refresh_access_token(self) -> None:
|
||||
"""Use MSAL to exchange the refresh token for a fresh access token.
|
||||
|
||||
Updates ``self._access_token`` and ``self._credentials_info`` in-place.
|
||||
|
||||
Raises:
|
||||
RuntimeError: MSAL reports an auth error.
|
||||
"""
|
||||
import msal
|
||||
|
||||
app = msal.ConfidentialClientApplication(
|
||||
client_id=settings.MS_CLIENT_ID,
|
||||
client_credential=settings.MS_CLIENT_SECRET,
|
||||
authority=f"https://login.microsoftonline.com/{settings.MS_TENANT_ID}",
|
||||
)
|
||||
scopes: list[str] = self._credentials_info.get("scope", "").split()
|
||||
if not scopes:
|
||||
scopes = ["https://graph.microsoft.com/.default"]
|
||||
|
||||
result = app.acquire_token_by_refresh_token(
|
||||
self._refresh_token,
|
||||
scopes=scopes,
|
||||
)
|
||||
if "access_token" not in result:
|
||||
error = result.get("error_description", result.get("error", "unknown"))
|
||||
raise RuntimeError(f"MS Graph token refresh failed: {error}")
|
||||
|
||||
self._access_token = result["access_token"]
|
||||
# MSAL may issue a new refresh token.
|
||||
if "refresh_token" in result:
|
||||
self._refresh_token = result["refresh_token"]
|
||||
self._credentials_info["refresh_token"] = result["refresh_token"]
|
||||
self._credentials_info["access_token"] = self._access_token
|
||||
|
||||
@property
|
||||
def refreshed_credentials(self) -> dict[str, Any] | None:
|
||||
"""Return updated credential dict if the access token was refreshed.
|
||||
|
||||
Returns ``None`` if no change was made.
|
||||
"""
|
||||
if self._access_token != self._original_access_token:
|
||||
return {**self._credentials_info, "access_token": self._access_token}
|
||||
return None
|
||||
|
||||
# ── HTTP helpers ───────────────────────────────────────────────────────
|
||||
|
||||
async def _get(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
url: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
*,
|
||||
retry_on_401: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""GET *url* with auth; refresh token on 401 and retry once."""
|
||||
resp = await client.get(url, params=params, headers=self._auth_headers())
|
||||
if resp.status_code == 401 and retry_on_401 and self._refresh_token:
|
||||
logger.debug("ms_graph: 401 on %s — refreshing token", url)
|
||||
await self._refresh_access_token()
|
||||
resp = await client.get(url, params=params, headers=self._auth_headers())
|
||||
if resp.status_code == 429:
|
||||
raise RuntimeError("MS Graph rate limit hit (429). Try again later.")
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
# ── Public API ─────────────────────────────────────────────────────────
|
||||
|
||||
async def fetch_emails(
|
||||
self,
|
||||
filter_config: dict[str, Any] | None = None,
|
||||
since: datetime | None = None,
|
||||
) -> list[EmailMessage]:
|
||||
"""Return up to ``_MAX_EMAILS`` Outlook messages matching *filter_config*.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filter_config:
|
||||
Optional dict with ``senders``, ``date_range``, ``folders`` keys.
|
||||
since:
|
||||
Hard lower-bound on email date (from last agent run).
|
||||
"""
|
||||
odata_filter = _build_email_filter(filter_config, since)
|
||||
params: dict[str, Any] = {
|
||||
"$top": 50,
|
||||
"$select": "id,subject,from,receivedDateTime,body,bodyPreview",
|
||||
"$orderby": "receivedDateTime desc",
|
||||
}
|
||||
if odata_filter:
|
||||
params["$filter"] = odata_filter
|
||||
|
||||
emails: list[EmailMessage] = []
|
||||
url = f"{_GRAPH_BASE}/me/messages"
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
while url and len(emails) < _MAX_EMAILS:
|
||||
data = await self._get(client, url, params if url.startswith(_GRAPH_BASE) else None)
|
||||
for item in data.get("value", []):
|
||||
emails.append(self._parse_email(item))
|
||||
if len(emails) >= _MAX_EMAILS:
|
||||
break
|
||||
url = data.get("@odata.nextLink", "")
|
||||
params = {} # nextLink already contains encoded params.
|
||||
|
||||
logger.info("ms_graph: fetched %d Outlook email(s)", len(emails))
|
||||
return emails
|
||||
|
||||
async def fetch_messages(
|
||||
self,
|
||||
filter_config: dict[str, Any] | None = None,
|
||||
since: datetime | None = None,
|
||||
) -> list[ChatMessage]:
|
||||
"""Return up to ``_MAX_MESSAGES`` Teams messages matching *filter_config*.
|
||||
|
||||
Fetches from ``/me/chats/getAllMessages`` (personal + group chats).
|
||||
The ``filter_config.channels`` key is checked as a text-filter on
|
||||
the channel name post-fetch (the API doesn't support channel OData
|
||||
filter directly on ``getAllMessages``).
|
||||
"""
|
||||
cfg = filter_config or {}
|
||||
channel_filter: list[str] = [c.lower() for c in cfg.get("channels", [])]
|
||||
params: dict[str, Any] = {"$top": 50}
|
||||
if since:
|
||||
params["$filter"] = f"createdDateTime ge {_odata_datetime(since)}"
|
||||
|
||||
messages: list[ChatMessage] = []
|
||||
url = f"{_GRAPH_BASE}/me/chats/getAllMessages"
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
while url and len(messages) < _MAX_MESSAGES:
|
||||
try:
|
||||
data = await self._get(client, url, params if url.startswith(_GRAPH_BASE) else None)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
# getAllMessages requires specific licensing; degrade gracefully.
|
||||
if exc.response.status_code in (403, 404):
|
||||
logger.warning(
|
||||
"ms_graph: /me/chats/getAllMessages not available (%d) — "
|
||||
"check Teams license or permissions",
|
||||
exc.response.status_code,
|
||||
)
|
||||
break
|
||||
raise
|
||||
|
||||
for item in data.get("value", []):
|
||||
msg = self._parse_teams_message(item)
|
||||
if channel_filter and msg.channel:
|
||||
if not any(c in msg.channel.lower() for c in channel_filter):
|
||||
continue
|
||||
messages.append(msg)
|
||||
if len(messages) >= _MAX_MESSAGES:
|
||||
break
|
||||
url = data.get("@odata.nextLink", "")
|
||||
params = {}
|
||||
|
||||
logger.info("ms_graph: fetched %d Teams message(s)", len(messages))
|
||||
return messages
|
||||
|
||||
# ── Parsers ────────────────────────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _parse_email(item: dict[str, Any]) -> EmailMessage:
|
||||
subject: str = item.get("subject", "(no subject)") or "(no subject)"
|
||||
sender_block = item.get("from", {}) or {}
|
||||
sender_addr = (
|
||||
(sender_block.get("emailAddress") or {}).get("address", "unknown")
|
||||
)
|
||||
date_str: str = item.get("receivedDateTime", "")
|
||||
try:
|
||||
date = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
date = datetime.now(timezone.utc)
|
||||
|
||||
body_block = item.get("body", {}) or {}
|
||||
content_type: str = body_block.get("contentType", "text")
|
||||
raw_body: str = body_block.get("content", "")
|
||||
if content_type == "html":
|
||||
body_text = _strip_html(raw_body)
|
||||
else:
|
||||
body_text = raw_body or item.get("bodyPreview", "")
|
||||
body_text = body_text[:_BODY_TRUNCATE]
|
||||
|
||||
return EmailMessage(
|
||||
id=item.get("id", ""),
|
||||
subject=subject,
|
||||
sender=sender_addr,
|
||||
body_text=body_text,
|
||||
date=date,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_teams_message(item: dict[str, Any]) -> ChatMessage:
|
||||
msg_id: str = item.get("id", "")
|
||||
sender_block = (item.get("from") or {}).get("user") or {}
|
||||
sender: str = sender_block.get("displayName", "unknown")
|
||||
channel: str | None = (item.get("channelIdentity") or {}).get("channelId")
|
||||
|
||||
date_str: str = item.get("createdDateTime", "")
|
||||
try:
|
||||
date = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
date = datetime.now(timezone.utc)
|
||||
|
||||
body_block = item.get("body", {}) or {}
|
||||
content_type: str = body_block.get("contentType", "text")
|
||||
raw_content: str = body_block.get("content", "")
|
||||
content = _strip_html(raw_content) if content_type == "html" else raw_content
|
||||
content = content[:_BODY_TRUNCATE]
|
||||
|
||||
return ChatMessage(
|
||||
id=msg_id,
|
||||
content=content,
|
||||
sender=sender,
|
||||
channel=channel,
|
||||
date=date,
|
||||
)
|
||||
Reference in New Issue
Block a user