") + + @staticmethod + def _attrs(tag: str) -> dict[str, str]: + return {m.group(1).lower(): html.unescape(m.group(2) if m.group(2) is not None else m.group(3)) for m in ChatHtmlSanitizer._ATTR.finditer(tag)} + + @staticmethod + def _size(value) -> int | None: + try: + n = round(float(value)) + except (TypeError, ValueError): + return None + return n if 0 < n <= 4096 else None + + @staticmethod + def _https_host(url: str, hosts: frozenset) -> bool: + try: + parts = urlsplit(url or "") + port = parts.port + except ValueError: + return False + return parts.scheme == "https" and port in (None, 443) and not parts.username and not parts.password and (parts.hostname or "").lower() in hosts + + @staticmethod + def is_gif_host(url: str) -> bool: + return ChatHtmlSanitizer._https_host(url, ChatHtmlSanitizer.GIF_HOSTS) + + @staticmethod + def hosted_content_id(src: str) -> str | None: + """The hosted-content id when `src` is exactly a Graph hosted-content URL, else None.""" + if not ChatHtmlSanitizer._https_host(src, frozenset({ChatHtmlSanitizer.GRAPH_HOST})): + return None + m = ChatHtmlSanitizer._HOSTED_PATH.fullmatch(urlsplit(src).path) + return m.group(1) if m else None + + @staticmethod + def media(message: dict) -> list[dict]: + """Images the body shows, in order: [{kind, url, hostedContentId, alt, width, height}]. + + kind "hosted": a pasted image held by Graph; `url` is null and `hostedContentId` is the id the web + passes to the plugin's media proxy. kind "gif": a Giphy/Tenor GIF; `url` is the public media URL + (exact allowlisted hosts, https only). Other images (emoji, external) are left out.""" + content = (message.get("body") or {}).get("content") or "" + out, seen = [], set() + for tag in ChatHtmlSanitizer._IMG_TAG.findall(content): + a = ChatHtmlSanitizer._attrs(tag) + src = a.get("src") or "" + if src in seen: + continue + hosted = ChatHtmlSanitizer.hosted_content_id(src) + if hosted: + item = {"kind": "hosted", "url": None, "hostedContentId": hosted} + elif ChatHtmlSanitizer.is_gif_host(src): + item = {"kind": "gif", "url": src, "hostedContentId": None} + else: + continue + seen.add(src) + item.update({"alt": (a.get("alt") or "")[:300] or None, "width": ChatHtmlSanitizer._size(a.get("width")), "height": ChatHtmlSanitizer._size(a.get("height"))}) + out.append(item) + return out + + @staticmethod + def reference_files(message: dict) -> list[dict]: + """Files shared into the chat (reference attachments): [{id, name, contentUrl}] (type and size are filled in separately).""" + out = [] + for a in message.get("attachments") or []: + if a.get("contentType") == "reference" and a.get("contentUrl"): + out.append({"id": a.get("id"), "name": a.get("name"), "contentUrl": a.get("contentUrl")}) + return out + + @staticmethod + def cards(message: dict) -> list[dict]: + """Card attachments (adaptive cards, polls, sticker apps, link previews): [{id, contentType, teamsAppId, name, content, sticker}]. + + `content` is the parsed card JSON for a read-only render (null when it isn't JSON or is too large); + `sticker` is {url, alt} when the card is a single image (sticker apps send those), else null.""" + out = [] + for a in message.get("attachments") or []: + ctype = a.get("contentType") or "" + if not ctype.startswith(ChatHtmlSanitizer.CARD_PREFIX): + continue + raw = a.get("content") + parsed = None + if isinstance(raw, str) and len(raw) <= ChatHtmlSanitizer.MAX_CARD_CHARS: + try: + parsed = json.loads(raw) + except ValueError: + parsed = None + elif isinstance(raw, dict): + parsed = raw + out.append({"id": a.get("id"), "contentType": ctype, "teamsAppId": a.get("teamsAppId"), "name": a.get("name"), "content": parsed, "sticker": ChatHtmlSanitizer._sticker(parsed)}) + return out + + @staticmethod + def _sticker(card) -> dict | None: + if not isinstance(card, dict) or card.get("type") != "AdaptiveCard": + return None + body = card.get("body") + if not isinstance(body, list) or len(body) != 1 or not isinstance(body[0], dict) or body[0].get("type") != "Image": + return None + url = body[0].get("url") + if not isinstance(url, str) or not (ChatHtmlSanitizer._https_host(url, ChatHtmlSanitizer.STICKER_HOSTS) or ChatHtmlSanitizer.is_gif_host(url)): + return None + return {"url": url, "alt": body[0].get("altText")} + + @staticmethod + def gif_html(url: str, alt: str | None, width: int | None, height: int | None) -> str: + """The markup Teams itself sends for a GIF picked from its GIF tab.""" + size = "".join(f' {k}="{v}"' for k, v in (("width", width), ("height", height)) if v) + return f'
" if self.write else "
" + attrs = [] + if tag == "a": + href = OneNoteHtmlSanitizer.safe_href(node.attrs.get("href")) + if href: + attrs.append(f'href="{html.escape(href, quote=True)}"') + if not self.write: + attrs.append('rel="noopener noreferrer"') + if tag in ("td", "th"): + for name in ("colspan", "rowspan"): + value = node.attrs.get(name, "") + if value.isdigit() and 1 <= int(value) <= 100: + attrs.append(f'{name}="{int(value)}"') + tag_value = node.attrs.get("data-tag", "").lower() + if tag_value and OneNoteHtmlSanitizer.TAG_VALUE.fullmatch(tag_value) and tag in ("p", "li", "span", "div", "h1", "h2", "h3", "h4", "h5", "h6"): + attrs.append(f'data-tag="{tag_value}"') + style = OneNoteHtmlSanitizer._style(node.attrs.get("style"), OneNoteHtmlSanitizer.WRITE_STYLE_PROPS if self.write else OneNoteHtmlSanitizer.STYLE_PROPS) + if style: + attrs.append(f'style="{html.escape(style, quote=True)}"') + head = f"<{tag} {' '.join(attrs)}>" if attrs else f"<{tag}>" + return f"{head}{self.children(node)}" + + def _position(self, node: _Node, positioned: bool, width: float | None = None) -> str: + if not positioned: + return "" + style = node.attrs.get("style") + left, top = OneNoteHtmlSanitizer._length(style, "left") or 0.0, OneNoteHtmlSanitizer._length(style, "top") or 0.0 + parts = [f"position:absolute;left:{OneNoteHtmlSanitizer._num(left)}px;top:{OneNoteHtmlSanitizer._num(top)}px"] + if width is not None: + parts.append(f"width:{OneNoteHtmlSanitizer._num(width)}px") + return f' style="{";".join(parts)}"' + + def image(self, node: _Node, positioned: bool) -> str: + rid = OneNoteHtmlSanitizer.resource_id(node.attrs.get("data-fullres-src")) or OneNoteHtmlSanitizer.resource_id(node.attrs.get("src")) + if rid is None: + return "" + sid = OneNoteHtmlSanitizer.scoped(self.scope, rid) + kind = (node.attrs.get("data-fullres-src-type") or node.attrs.get("data-src-type") or "").lower() + self.resources.setdefault(sid, {"id": sid, "kind": "image", "name": None, "contentType": kind if OneNoteHtmlSanitizer.CONTENT_TYPE.fullmatch(kind) else None}) + alt = (node.attrs.get("alt") or "")[:OneNoteHtmlSanitizer.MAX_ALT] + width, height = OneNoteHtmlSanitizer._attr_number(node.attrs.get("width")), OneNoteHtmlSanitizer._attr_number(node.attrs.get("height")) + size = (f' width="{OneNoteHtmlSanitizer._num(width)}"' if width else "") + (f' height="{OneNoteHtmlSanitizer._num(height)}"' if height else "") + return f'
{OneNoteHtmlSanitizer.sanitize_for_write(body)}
" if body and body.strip() else ""
+ return f"") - - -# --- what Graph returns ------------------------------------------------------ -_IMG_TAG = re.compile(r"
`
-in the message body, so a GIF picked here sends the same way. The Teams clients query Giphy with
-Microsoft's own key; Schuly needs its own (`GIPHY_API_KEY`, free from developers.giphy.com).
-Without one the search answers 503.
-"""
-
-import os
-import time
-from urllib.parse import urlencode
-
-from entrance import http_client
-
-from src import chat_html
-from src.account_cache import AccountCache, for_broker
-
-API = "https://api.giphy.com/v1/gifs"
-RATING = os.getenv("GIPHY_RATING", "g")
-MAX_LIMIT = 50
-RESULTS_TTL = 300 # identical searches within 5 minutes share one Giphy call
-RATE_PER_MINUTE = 30
-_results = AccountCache(max_items=500)
-
-
-class GifSearchUnavailable(RuntimeError):
- pass
-
-
-def api_key() -> str | None:
- return os.getenv("GIPHY_API_KEY") or None
-
-
-def _image(images: dict, *names: str) -> dict:
- for name in names:
- img = images.get(name) or {}
- if img.get("url"):
- return img
- return {}
-
-
-def _int(value) -> int | None:
- try:
- return int(value)
- except (TypeError, ValueError):
- return None
-
-
-def _shape(item: dict) -> dict | None:
- images = item.get("images") or {}
- full = _image(images, "downsized_medium", "downsized", "original")
- preview = _image(images, "fixed_width_small", "fixed_width", "preview_gif") or full
- url = (full.get("url") or "").split("?")[0]
- if not chat_html.is_gif_host(url):
- return None
- return {"id": item.get("id"), "title": item.get("title") or "GIF", "url": url, "previewUrl": (preview.get("url") or url).split("?")[0], "width": _int(full.get("width")), "height": _int(full.get("height"))}
-
-
-def allow(broker) -> bool:
- """A fixed one-minute window per account: at most RATE_PER_MINUTE searches."""
- cache = for_broker(broker)
- window = ("gif_rate", int(time.time() // 60))
- with cache.lock("gif_rate"):
- used = cache.get(window, 0)
- if used >= RATE_PER_MINUTE:
- return False
- cache.set(window, used + 1, 120)
- return True
-
-
-def search(query: str | None, limit: int) -> list[dict]:
- key = api_key()
- if not key:
- raise GifSearchUnavailable("GIF search needs a Giphy API key (GIPHY_API_KEY).")
- cache_key = ((query or "").strip().lower() or None, limit)
- hit = _results.get(cache_key)
- if hit is not None:
- return hit
- found = _search(key, query, limit)
- _results.set(cache_key, found, RESULTS_TTL)
- return found
-
-
-def _search(key: str, query: str | None, limit: int) -> list[dict]:
- params = {"api_key": key, "limit": max(1, min(limit, MAX_LIMIT)), "rating": RATING}
- if query:
- params["q"] = query
- r = http_client().request("GET", f"{API}/{'search' if query else 'trending'}?{urlencode(params)}", timeout=15)
- if r.status_code != 200:
- raise RuntimeError(f"Giphy answered {r.status_code}")
- return [g for g in (_shape(i) for i in r.json().get("data") or []) if g]
diff --git a/src/infrastructure/files/file_text_extractor.py b/src/infrastructure/files/file_text_extractor.py
new file mode 100644
index 0000000..247d5e5
--- /dev/null
+++ b/src/infrastructure/files/file_text_extractor.py
@@ -0,0 +1,292 @@
+"""Plain text out of documents, spreadsheets, presentations, PDFs and text files (the FileText answer).
+
+Office and PDF libraries are imported lazily. Untrusted bytes are checked before they reach an OOXML library: the source is capped,
+the zip's declared uncompressed size is summed (zipfile never reads past a member's declared size) and a member with an absurd
+compression ratio or an XML part declaring entities is refused. Nothing here logs content; it runs in the worker thread.
+"""
+
+import csv
+import io
+import posixpath
+import zipfile
+from datetime import date, datetime, time
+
+from src.application.abstractions.i_file_text_extractor import IFileTextExtractor
+from src.domain.errors.text_error import TextError
+from src.domain.files.file_text_limits import FileTextLimits
+
+
+class FileTextExtractor(IFileTextExtractor):
+ """Plain text out of documents, spreadsheets, presentations, PDFs and text files."""
+
+ MAX_SOURCE_BYTES = FileTextLimits.MAX_SOURCE_BYTES
+ MAX_UNCOMPRESSED = 200 * 1024 * 1024
+ MAX_ZIP_MEMBERS = 10000
+ MAX_RATIO = 250
+ RATIO_FROM = 4 * 1024 * 1024
+ DEFAULT_CHARS = FileTextLimits.DEFAULT_CHARS
+ MAX_CHARS = FileTextLimits.MAX_CHARS
+ TEXT_CAP = 2_000_000
+ MAX_PDF_PAGES = 300
+ MAX_SLIDES = 500
+ DEFAULT_ROWS = FileTextLimits.DEFAULT_ROWS
+ MAX_ROWS = FileTextLimits.MAX_ROWS
+ MAX_ROW_OFFSET = FileTextLimits.MAX_ROW_OFFSET
+ PEEK_ROWS = 10000
+ DOCUMENT = {"docx"}
+ SPREADSHEET = {"xlsx", "xlsm"}
+ PRESENTATION = {"pptx"}
+ PDF = {"pdf"}
+ TEXT = {"txt", "md", "markdown", "csv", "tsv", "json", "xml", "html", "htm", "ics", "log", "yaml", "yml", "ini", "cfg", "conf", "toml", "tex", "sql", "css", "js", "ts", "py", "java", "cs", "c", "cpp", "h", "sh", "ps1", "rb", "go", "rs", "php", "kt", "swift", "dart", "lua", "vcf", "srt"}
+ LEGACY = {"doc", "xls", "ppt", "odt", "ods", "odp", "rtf"}
+ UNSUPPORTED = FileTextLimits.UNSUPPORTED
+
+ def extension(self, name) -> str:
+ return posixpath.splitext(name or "")[1].lstrip(".").lower() if isinstance(name, str) else ""
+
+ def kind_of(self, name) -> str | None:
+ """document | spreadsheet | presentation | pdf | text | legacy (convert to PDF first), or None when the type can't be read."""
+ ext = self.extension(name)
+ for kind, exts in (("document", self.DOCUMENT), ("spreadsheet", self.SPREADSHEET), ("presentation", self.PRESENTATION), ("pdf", self.PDF), ("text", self.TEXT), ("legacy", self.LEGACY)):
+ if ext in exts:
+ return kind
+ return None
+
+ def extract(self, name: str, data: bytes, offset: int = 0, max_chars: int = DEFAULT_CHARS, sheet: str | None = None, row_offset: int = 0, max_rows: int = DEFAULT_ROWS, notes: list[str] | None = None) -> dict:
+ """The FileText answer for `data`. Legacy formats must be converted to PDF by the caller (`extract` then gets the PDF, name ending in .pdf)."""
+ kind = self.kind_of(name)
+ if kind is None or kind == "legacy":
+ raise TextError(415, self.UNSUPPORTED)
+ if len(data) > self.MAX_SOURCE_BYTES:
+ raise TextError(413, "The file is larger than 25 MB.")
+ max_chars = max(1, min(max_chars, self.MAX_CHARS))
+ offset = max(0, offset)
+ notes = list(notes or [])
+ result = {"name": name, "kind": kind}
+ try:
+ if kind == "spreadsheet":
+ self._zip_guard(data)
+ text, extra = self._spreadsheet(data, sheet, max(0, min(row_offset, self.MAX_ROW_OFFSET)), max(1, min(max_rows, self.MAX_ROWS)), notes)
+ elif kind == "document":
+ self._zip_guard(data)
+ text, extra = self._document(data, notes), {}
+ elif kind == "presentation":
+ self._zip_guard(data)
+ text, extra = self._presentation(data, notes, result), {}
+ elif kind == "pdf":
+ text, extra = self._pdf(data, notes, result), {}
+ else:
+ text, extra = self._plain(name, data, notes), {}
+ except TextError:
+ raise
+ except Exception:
+ raise TextError(422, "The file could not be read.")
+ capped = len(text) > self.TEXT_CAP
+ if capped:
+ text = text[:self.TEXT_CAP]
+ notes.append(f"Only the first {self.TEXT_CAP} characters are counted.")
+ page = text[offset : offset + max_chars]
+ end = offset + len(page)
+ more = end < len(text)
+ result.update({"text": page, "offset": offset, "nextOffset": end if more else None, "totalChars": len(text), "truncated": more or capped or extra.pop("limited", False), "notes": notes})
+ result.update(extra)
+ return result
+
+ def _zip_guard(self, data: bytes):
+ try:
+ archive = zipfile.ZipFile(io.BytesIO(data))
+ except zipfile.BadZipFile:
+ raise TextError(422, "The file is damaged or not an Office document.")
+ with archive:
+ members = archive.infolist()
+ if len(members) > self.MAX_ZIP_MEMBERS:
+ raise TextError(413, "The file contains too many parts.")
+ total = 0
+ for m in members:
+ total += m.file_size
+ if m.file_size >= self.RATIO_FROM and m.file_size > self.MAX_RATIO * max(m.compress_size, 1):
+ raise TextError(413, "The file is compressed suspiciously well and is refused.")
+ if total > self.MAX_UNCOMPRESSED:
+ raise TextError(413, "The file unpacks to more than 200 MB.")
+ for m in members:
+ if m.filename.lower().endswith((".xml", ".rels")) and m.file_size:
+ with archive.open(m) as part:
+ head = part.read(4096).upper()
+ if b" str:
+ return " ".join(str(text).split()).replace("|", "\\|")
+
+ def _document(self, data: bytes, notes: list[str]) -> str:
+ from docx import Document
+ from docx.table import Table
+ from docx.text.paragraph import Paragraph
+
+ doc = Document(io.BytesIO(data))
+ out, size = [], 0
+ for child in doc.element.body.iterchildren():
+ tag = child.tag.rsplit("}", 1)[-1]
+ if tag == "p":
+ para = Paragraph(child, doc)
+ text = para.text
+ if not text.strip():
+ out.append("")
+ continue
+ style = (para.style.name if para.style is not None else "") or ""
+ if style == "Title":
+ text = "# " + text
+ elif style.startswith("Heading "):
+ level = style[8:]
+ text = "#" * (int(level) if level.isdigit() and 1 <= int(level) <= 6 else 1) + " " + text
+ out.append(text)
+ size += len(text)
+ elif tag == "tbl":
+ for row in Table(child, doc).rows:
+ seen, cells = set(), []
+ for cell in row.cells:
+ if id(cell._tc) in seen:
+ continue
+ seen.add(id(cell._tc))
+ cells.append(self._cell(cell.text))
+ line = "| " + " | ".join(cells) + " |"
+ out.append(line)
+ size += len(line)
+ if size > self.TEXT_CAP:
+ notes.append("The document is longer than the reader's limit; the rest is left out.")
+ break
+ return self._squeeze("\n".join(out))
+
+ def _squeeze(self, text: str) -> str:
+ while "\n\n\n" in text:
+ text = text.replace("\n\n\n", "\n\n")
+ return text.strip()
+
+ def _value(self, v) -> str:
+ if v is None:
+ return ""
+ if isinstance(v, (datetime, date, time)):
+ return v.isoformat()
+ if isinstance(v, bool):
+ return "TRUE" if v else "FALSE"
+ return str(v)
+
+ def _spreadsheet(self, data: bytes, sheet: str | None, row_offset: int, max_rows: int, notes: list[str]):
+ import defusedxml # noqa: F401 (openpyxl parses with defusedxml when it is installed)
+ from itertools import islice
+
+ from openpyxl import load_workbook
+
+ wb = load_workbook(io.BytesIO(data), read_only=True, data_only=True)
+ try:
+ sheets = [{"name": ws.title, "rows": int(ws.max_row or 0), "columns": int(ws.max_column or 0)} for ws in wb.worksheets]
+ if not sheets:
+ raise TextError(422, "The workbook has no sheets.")
+ name = sheet if sheet is not None else sheets[0]["name"]
+ if name not in wb.sheetnames:
+ raise TextError(404, "Sheet not found.")
+ ws = wb[name]
+ rows = ws.iter_rows(values_only=True)
+ for _ in islice(rows, row_offset):
+ pass
+ page = [list(r) for r in islice(rows, max_rows)]
+ more = False
+ for peeked in islice(rows, self.PEEK_ROWS):
+ if any(v is not None and v != "" for v in peeked):
+ more = True
+ break
+ while page and all(v is None or v == "" for v in page[-1]):
+ page.pop()
+ width = max((max((i + 1 for i, v in enumerate(r) if v is not None and v != ""), default=0) for r in page), default=0)
+ buf = io.StringIO()
+ writer = csv.writer(buf, lineterminator="\n")
+ for r in page:
+ writer.writerow([self._value(v) for v in (r + [None] * width)[:width]])
+ count = len(page)
+ if more:
+ notes.append(f"Only rows {row_offset + 1} to {row_offset + count} of sheet {name} are included; continue with rowOffset={row_offset + count}.")
+ extra = {"sheets": sheets, "sheet": name, "rowOffset": row_offset, "rowCount": count, "nextRowOffset": row_offset + count if more else None, "limited": more}
+ return buf.getvalue(), extra
+ finally:
+ wb.close()
+
+ def _shape_text(self, shape, out: list[str]):
+ if getattr(shape, "has_text_frame", False) and shape.has_text_frame:
+ text = shape.text_frame.text.strip()
+ if text:
+ out.append(text)
+ if getattr(shape, "has_table", False) and shape.has_table:
+ for row in shape.table.rows:
+ out.append("| " + " | ".join(self._cell(c.text) for c in row.cells) + " |")
+ if shape.shape_type == 6: # group
+ for inner in shape.shapes:
+ self._shape_text(inner, out)
+
+ def _presentation(self, data: bytes, notes: list[str], result: dict) -> str:
+ from pptx import Presentation
+
+ prs = Presentation(io.BytesIO(data))
+ slides = list(prs.slides)
+ result["pages"] = len(slides)
+ out, size = [], 0
+ for number, slide in enumerate(slides, 1):
+ if number > self.MAX_SLIDES:
+ notes.append(f"Only the first {self.MAX_SLIDES} slides are included.")
+ break
+ lines = [f"--- Slide {number} ---"]
+ for shape in slide.shapes:
+ self._shape_text(shape, lines)
+ if slide.has_notes_slide:
+ spoken = slide.notes_slide.notes_text_frame.text.strip() if slide.notes_slide.notes_text_frame is not None else ""
+ if spoken:
+ lines.append("Notes: " + spoken)
+ block = "\n".join(lines)
+ out.append(block)
+ size += len(block)
+ if size > self.TEXT_CAP:
+ notes.append("The presentation is longer than the reader's limit; the rest is left out.")
+ break
+ return "\n\n".join(out)
+
+ def _pdf(self, data: bytes, notes: list[str], result: dict) -> str:
+ from pypdf import PdfReader
+
+ reader = PdfReader(io.BytesIO(data))
+ if reader.is_encrypted:
+ try:
+ ok = reader.decrypt("")
+ except Exception:
+ ok = 0
+ if not ok:
+ raise TextError(422, "The PDF is password protected.")
+ total = len(reader.pages)
+ result["pages"] = total
+ out, size = [], 0
+ for number, page in enumerate(reader.pages, 1):
+ if number > self.MAX_PDF_PAGES:
+ notes.append(f"Only the first {self.MAX_PDF_PAGES} pages are included.")
+ break
+ out.append(f"--- Page {number} ---\n{(page.extract_text() or '').strip()}")
+ size += len(out[-1])
+ if size > self.TEXT_CAP:
+ notes.append("The PDF is longer than the reader's limit; the rest is left out.")
+ break
+ return "\n\n".join(out)
+
+ def decode(self, data: bytes) -> str:
+ if data.startswith(b"\xef\xbb\xbf"):
+ return data[3:].decode("utf-8", "replace")
+ if data.startswith((b"\xff\xfe", b"\xfe\xff")):
+ return data.decode("utf-16", "replace")
+ try:
+ return data.decode("utf-8")
+ except UnicodeDecodeError:
+ return data.decode("latin-1")
+
+ def _plain(self, name: str, data: bytes, notes: list[str]) -> str:
+ text = self.decode(data)
+ if self.extension(name) in ("html", "htm"):
+ from outnote import html_to_text
+
+ return html_to_text(text)
+ return text.replace("\r\n", "\n").replace("\r", "\n")
diff --git a/src/infrastructure/giphy/gif_search.py b/src/infrastructure/giphy/gif_search.py
new file mode 100644
index 0000000..2410953
--- /dev/null
+++ b/src/infrastructure/giphy/gif_search.py
@@ -0,0 +1,86 @@
+"""GIF search over Giphy's public API.
+
+Teams shows Giphy GIFs as a plain ``
+in the message body, so a GIF picked here sends the same way. The Teams clients query Giphy with
+Microsoft's own key; Schuly needs its own (`GIPHY_API_KEY`, free from developers.giphy.com).
+Without one the search answers 503.
+"""
+
+import time
+from urllib.parse import urlencode
+from entrance import http_client
+
+from src.application.abstractions.i_gif_search import IGifSearch
+from src.application.options.giphy_options import GiphyOptions
+from src.application.services.chat_html_sanitizer import ChatHtmlSanitizer
+from src.domain.errors.gif_search_unavailable import GifSearchUnavailable
+from src.infrastructure.microsoft.account_cache import AccountCache
+
+
+class GifSearch(IGifSearch):
+ """GIF search over Giphy's public API, with a per-account rate limit."""
+
+ API = "https://api.giphy.com/v1/gifs"
+ MAX_LIMIT = 50
+ RESULTS_TTL = 300 # identical searches within 5 minutes share one Giphy call
+ RATE_PER_MINUTE = 30
+ http_client = staticmethod(http_client)
+
+ def __init__(self, html: ChatHtmlSanitizer, options: GiphyOptions):
+ self._html = html
+ self._options = options
+ self._results = AccountCache(max_items=500)
+
+ def _image(self, images: dict, *names: str) -> dict:
+ for name in names:
+ img = images.get(name) or {}
+ if img.get("url"):
+ return img
+ return {}
+
+ def _int(self, value) -> int | None:
+ try:
+ return int(value)
+ except (TypeError, ValueError):
+ return None
+
+ def _shape(self, item: dict) -> dict | None:
+ images = item.get("images") or {}
+ full = self._image(images, "downsized_medium", "downsized", "original")
+ preview = self._image(images, "fixed_width_small", "fixed_width", "preview_gif") or full
+ url = (full.get("url") or "").split("?")[0]
+ if not self._html.is_gif_host(url):
+ return None
+ return {"id": item.get("id"), "title": item.get("title") or "GIF", "url": url, "previewUrl": (preview.get("url") or url).split("?")[0], "width": self._int(full.get("width")), "height": self._int(full.get("height"))}
+
+ def allow(self, broker) -> bool:
+ """A fixed one-minute window per account: at most RATE_PER_MINUTE searches."""
+ cache = AccountCache.for_broker(broker)
+ window = ("gif_rate", int(time.time() // 60))
+ with cache.lock("gif_rate"):
+ used = cache.get(window, 0)
+ if used >= self.RATE_PER_MINUTE:
+ return False
+ cache.set(window, used + 1, 120)
+ return True
+
+ def search(self, query: str | None, limit: int) -> list[dict]:
+ key = self._options.api_key()
+ if not key:
+ raise GifSearchUnavailable("GIF search needs a Giphy API key (GIPHY_API_KEY).")
+ cache_key = ((query or "").strip().lower() or None, limit)
+ hit = self._results.get(cache_key)
+ if hit is not None:
+ return hit
+ found = self._search(key, query, limit)
+ self._results.set(cache_key, found, self.RESULTS_TTL)
+ return found
+
+ def _search(self, key: str, query: str | None, limit: int) -> list[dict]:
+ params = {"api_key": key, "limit": max(1, min(limit, self.MAX_LIMIT)), "rating": self._options.rating}
+ if query:
+ params["q"] = query
+ r = self.http_client().request("GET", f"{self.API}/{'search' if query else 'trending'}?{urlencode(params)}", timeout=15)
+ if r.status_code != 200:
+ raise RuntimeError(f"Giphy answered {r.status_code}")
+ return [g for g in (self._shape(i) for i in r.json().get("data") or []) if g]
diff --git a/src/infrastructure/graph/channel_messages.py b/src/infrastructure/graph/channel_messages.py
new file mode 100644
index 0000000..b097b51
--- /dev/null
+++ b/src/infrastructure/graph/channel_messages.py
@@ -0,0 +1,50 @@
+"""Teams channel posts and their replies, normalized for the web and the MCP tools (read only)."""
+
+from html import unescape
+from outnote import html_to_text
+
+from src.application.abstractions.i_channel_messages import IChannelMessages
+from src.domain.validation.graph_ids import GraphIds
+
+
+class ChannelMessages(IChannelMessages):
+ """Teams channel posts and their replies, normalized."""
+
+ MAX_TEXT = 4000
+ PAGE = 50
+
+ def _text(self, body) -> str:
+ if not isinstance(body, dict):
+ return ""
+ content = body.get("content") or ""
+ text = unescape(content).strip() if body.get("contentType") == "text" else html_to_text(content)
+ return text[:self.MAX_TEXT]
+
+ def _sender(self, value):
+ if not isinstance(value, dict):
+ return None
+ who = value.get("user") or value.get("application") or value.get("device")
+ if not isinstance(who, dict):
+ return None
+ return {"displayName": who.get("displayName"), "id": who.get("id")}
+
+ def normalize(self, message: dict) -> dict | None:
+ """A ChannelMessage, or None for system messages (`messageType` != message) and deleted ones."""
+ if not isinstance(message, dict) or message.get("messageType") != "message" or message.get("deletedDateTime"):
+ return None
+ attachments = [{"name": a.get("name"), "contentType": a.get("contentType"), "webUrl": a.get("contentUrl")} for a in message.get("attachments") or [] if isinstance(a, dict) and a.get("name")]
+ return {"id": message.get("id"), "replyToId": message.get("replyToId"), "from": self._sender(message.get("from")), "createdDateTime": message.get("createdDateTime"), "lastModifiedDateTime": message.get("lastModifiedDateTime"), "subject": message.get("subject"), "text": self._text(message.get("body")), "attachments": attachments, "replyCount": None, "webUrl": message.get("webUrl"), "messageType": "message"}
+
+ def _rows(self, note, path: str, top: int) -> list[dict]:
+ rows = note.list(path, {"$top": min(top, self.PAGE)}, limit=top)
+ return [m for m in (self.normalize(r) for r in rows) if m]
+
+ def messages(self, note, team_id: str, channel_id: str, top: int) -> list[dict]:
+ """Top-level posts, newest first (follows nextLink until `top` raw posts were read)."""
+ out = self._rows(note, f"teams/{GraphIds.seg(team_id)}/channels/{GraphIds.seg(channel_id)}/messages", top)
+ return sorted(out, key=lambda m: m.get("createdDateTime") or "", reverse=True)
+
+ def replies(self, note, team_id: str, channel_id: str, message_id: str, top: int) -> list[dict]:
+ """Replies to one post, oldest first."""
+ out = self._rows(note, f"teams/{GraphIds.seg(team_id)}/channels/{GraphIds.seg(channel_id)}/messages/{GraphIds.seg(message_id)}/replies", top)
+ return sorted(out, key=lambda m: m.get("createdDateTime") or "")
diff --git a/src/infrastructure/graph/file_thumbnails.py b/src/infrastructure/graph/file_thumbnails.py
new file mode 100644
index 0000000..0bee8e3
--- /dev/null
+++ b/src/infrastructure/graph/file_thumbnails.py
@@ -0,0 +1,145 @@
+"""File kinds and thumbnails for files shared in chats and for OneDrive items.
+
+A file shared in a chat is a `reference` attachment whose `contentUrl` is a SharePoint / OneDrive link. Graph turns that link into
+its drive item (`shares/{u!base64url(url)}/driveItem`), and the item has thumbnails (`thumbnails/0/{small|medium|large}/content`) for
+Office documents, PDFs, images and videos. The resolved item ids are cached per account for an hour.
+
+Only SharePoint / OneDrive hosts are accepted (`*.sharepoint.com` by suffix, `onedrive.live.com` and `1drv.ms` exactly), https only.
+"""
+
+import posixpath
+import re
+from urllib.parse import quote, urlsplit
+from outnote import GraphError
+
+from src.application.abstractions.i_file_thumbnails import IFileThumbnails
+from src.application.services.graph_paths import GraphPaths
+from src.infrastructure.microsoft.account_cache import AccountCache
+
+
+class FileThumbnails(IFileThumbnails):
+ """File kinds, the SharePoint / OneDrive host check, and thumbnails of chat files and drive items."""
+
+ ITEM_TTL = 3600
+ SIZES = ("small", "medium", "large")
+ EXACT_HOSTS = frozenset({"onedrive.live.com", "1drv.ms"})
+ HOST_SUFFIXES = (".sharepoint.com",)
+ GRAPH_ID = re.compile(r"[A-Za-z0-9!=_\-+.]{1,512}")
+ MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024
+ KINDS_BY_EXTENSION = {
+ "word": {"doc", "docx", "docm", "dot", "dotx", "odt", "rtf"},
+ "excel": {"xls", "xlsx", "xlsm", "xlsb", "xlt", "xltx", "ods", "csv"},
+ "powerpoint": {"ppt", "pptx", "pptm", "pps", "ppsx", "odp"},
+ "pdf": {"pdf"},
+ "onenote": {"one", "onetoc2"},
+ "image": {"png", "jpg", "jpeg", "gif", "webp", "bmp", "svg", "heic", "tif", "tiff", "ico", "avif"},
+ "video": {"mp4", "mov", "mkv", "webm", "avi", "m4v", "wmv", "mpg", "mpeg"},
+ "audio": {"mp3", "wav", "m4a", "ogg", "oga", "flac", "aac", "wma", "opus"},
+ "archive": {"zip", "rar", "7z", "tar", "gz", "tgz", "bz2", "xz"},
+ "code": {"py", "js", "ts", "tsx", "jsx", "java", "cs", "c", "cpp", "h", "hpp", "go", "rs", "rb", "php", "sh", "ps1", "json", "xml", "yml", "yaml", "html", "htm", "css", "scss", "sql", "kt", "swift", "dart", "lua", "ipynb", "md"},
+ "text": {"txt", "log", "ini", "cfg", "conf", "tex"},
+ }
+ EXTENSION_KIND = {ext: kind for kind, exts in KINDS_BY_EXTENSION.items() for ext in exts}
+ THUMBNAIL_KINDS = frozenset({"word", "excel", "powerpoint", "pdf", "image", "video", "text"})
+
+ def is_share_url(self, url) -> bool:
+ """True only for an https SharePoint / OneDrive link (no credentials, default port)."""
+ if not isinstance(url, str) or not url or len(url) > 4000 or any(ord(c) < 33 for c in url):
+ return False
+ try:
+ parts = urlsplit(url)
+ port = parts.port
+ except ValueError:
+ return False
+ host = (parts.hostname or "").lower()
+ return parts.scheme == "https" and port in (None, 443) and not parts.username and not parts.password and (host in self.EXACT_HOSTS or any(host.endswith(s) for s in self.HOST_SUFFIXES))
+
+ def file_kind(self, name, mime) -> str:
+ """word | excel | powerpoint | pdf | onenote | image | video | audio | archive | code | text | other, from the extension, then the MIME type."""
+ ext = posixpath.splitext(name or "")[1].lstrip(".").lower() if isinstance(name, str) else ""
+ if ext in self.EXTENSION_KIND:
+ return self.EXTENSION_KIND[ext]
+ mime = (mime or "").split(";")[0].strip().lower() if isinstance(mime, str) else ""
+ major = mime.split("/")[0]
+ if major in ("image", "video", "audio"):
+ return major
+ if mime == "application/pdf":
+ return "pdf"
+ if "wordprocessingml" in mime or mime == "application/msword":
+ return "word"
+ if "spreadsheetml" in mime or mime == "application/vnd.ms-excel":
+ return "excel"
+ if "presentationml" in mime or mime == "application/vnd.ms-powerpoint":
+ return "powerpoint"
+ if mime in ("application/zip", "application/x-7z-compressed", "application/vnd.rar", "application/gzip", "application/x-tar"):
+ return "archive"
+ if mime in ("application/json", "application/xml", "application/javascript", "text/html", "text/css", "text/x-python"):
+ return "code"
+ if major == "text":
+ return "text"
+ return "other"
+
+ def has_thumbnail(self, url, kind: str) -> bool:
+ """Whether the file's host and type are ones that have thumbnails; decided from the data already at hand (no call per file)."""
+ return kind in self.THUMBNAIL_KINDS and self.is_share_url(url)
+
+ def image_type(self, head: bytes) -> str | None:
+ """JPEG, PNG or WEBP from the magic bytes; anything else is None."""
+ if head.startswith(b"\xff\xd8\xff"):
+ return "image/jpeg"
+ if head.startswith(b"\x89PNG\r\n\x1a\n"):
+ return "image/png"
+ if head[:4] == b"RIFF" and head[8:12] == b"WEBP":
+ return "image/webp"
+ return None
+
+ def safe_graph_id(self, value) -> bool:
+ return isinstance(value, str) and bool(self.GRAPH_ID.fullmatch(value)) and ".." not in value and value != "."
+
+ def _resolve(self, note, broker, url: str) -> tuple[str, str] | None:
+ """(drive id, item id) of the file behind a share URL, cached for an hour; None when it is gone, a folder or unreadable."""
+ cache = AccountCache.for_broker(broker)
+ key = ("share_item", url)
+ marker = object()
+ hit = cache.get(key, marker)
+ if hit is not marker:
+ return hit
+ try:
+ item = note.get(GraphPaths.drive_item_path(url, "id,parentReference,file,size,webUrl"))
+ except GraphError as ex:
+ if ex.status in (400, 403, 404):
+ cache.set(key, None, self.ITEM_TTL)
+ return None
+ raise
+ drive = ((item or {}).get("parentReference") or {}).get("driveId")
+ item_id = (item or {}).get("id")
+ found = (drive, item_id) if (item or {}).get("file") is not None and self.safe_graph_id(drive) and self.safe_graph_id(item_id) else None
+ cache.set(key, found, self.ITEM_TTL)
+ return found
+
+ def _content(self, note, path: str) -> bytes | None:
+ try:
+ data = note.get(path, raw="bytes")
+ except GraphError as ex:
+ if ex.status in (400, 404, 415):
+ return None
+ raise
+ if not data or len(data) > self.MAX_THUMBNAIL_BYTES or self.image_type(data[:12]) is None:
+ return None
+ return data
+
+ def chat_file_thumbnail(self, note, broker, url: str, size: str) -> bytes | None:
+ """The thumbnail bytes of a chat-shared file, or None when the URL isn't a share link or the file has no thumbnail."""
+ if size not in self.SIZES or not self.is_share_url(url):
+ return None
+ ids = self._resolve(note, broker, url)
+ if not ids:
+ return None
+ return self._content(note, f"drives/{quote(ids[0], safe='')}/items/{quote(ids[1], safe='')}/thumbnails/0/{size}/content")
+
+ def drive_item_thumbnail(self, note, item_id: str, size: str, drive_id: str | None = None) -> bytes | None:
+ """The thumbnail of an item in the user's OneDrive, or of `drive_id`'s item (a team library, a shared drive)."""
+ if size not in self.SIZES or not self.safe_graph_id(item_id) or (drive_id is not None and not self.safe_graph_id(drive_id)):
+ return None
+ base = f"drives/{quote(drive_id, safe='')}/items" if drive_id else "me/drive/items"
+ return self._content(note, f"{base}/{quote(item_id, safe='')}/thumbnails/0/{size}/content")
diff --git a/src/infrastructure/graph/graph_search.py b/src/infrastructure/graph/graph_search.py
new file mode 100644
index 0000000..06ec404
--- /dev/null
+++ b/src/infrastructure/graph/graph_search.py
@@ -0,0 +1,359 @@
+"""Search over files, mail, chats and channel posts with the Microsoft Search API, with the place each hit lives in.
+
+Files (`driveItem`) and messages (`message` + `chatMessage`) can't share one Search API request ("Invalid entity type combination"),
+so they run as two requests in parallel; a source that fails is reported in `errors` and never fails the search. When the Search API
+refuses the file search (400 / 403 / 5xx), the user's drive and each joined team's library are searched one by one instead. The query
+text is never logged; it only travels in the request body to Microsoft (and, for the drive fallback, in a Graph URL).
+"""
+
+import logging
+import re
+from concurrent.futures import ThreadPoolExecutor
+from html import unescape
+from urllib.parse import quote, unquote
+from outnote import GraphError
+
+from src.application.abstractions.i_graph_search import IGraphSearch
+from src.application.abstractions.i_teams_files import ITeamsFiles
+from src.infrastructure.microsoft.account_cache import AccountCache
+
+
+class Context:
+ """Lazily loaded lookups shared by the hits of one search; each failure only costs the detail it would have given."""
+
+ def __init__(self, note, broker, teams_files: ITeamsFiles):
+ self.note, self.cache = note, AccountCache.for_broker(broker)
+ self.broker = broker
+ self._teams_files = teams_files
+ self._own = self._drives = self._shared = None
+ self._teams = None
+ self._channels: dict[str, dict[str, str]] = {}
+
+ def teams(self) -> dict[str, str]:
+ if self._teams is None:
+ try:
+ self._teams = {t["id"]: t.get("displayName") or "" for t in self._teams_files.joined_teams(self.note, self.broker)}
+ except Exception as ex:
+ GraphSearch.log.info("search: joined teams failed: %s", type(ex).__name__)
+ self._teams = {}
+ return self._teams
+
+ def own_drive(self):
+ if self._own is None:
+ try:
+ self._own = self._teams_files.own_drive_id(self.note, self.broker) or ""
+ except Exception as ex:
+ GraphSearch.log.info("search: own drive failed: %s", type(ex).__name__)
+ self._own = ""
+ return self._own
+
+ def team_drives(self) -> dict[str, str]:
+ """Library drive id -> team id for the joined teams (cached 1 h)."""
+ if self._drives is None:
+ hit = self.cache.get(("search", "team_drives"))
+ if hit is None:
+ hit = {}
+ complete = True
+
+ def one(team_id):
+ try:
+ return team_id, self._teams_files.team_drive_id(self.note, self.broker, team_id)
+ except Exception:
+ return team_id, None
+
+ with ThreadPoolExecutor(max_workers=GraphSearch.FALLBACK_WORKERS) as pool:
+ for team_id, drive_id in pool.map(one, list(self.teams())):
+ if drive_id:
+ hit[drive_id] = team_id
+ else:
+ complete = False
+ self.cache.set(("search", "team_drives"), hit, GraphSearch.MAP_TTL if complete else 300)
+ self._drives = hit
+ return self._drives
+
+ def channel_names(self, team_id: str) -> dict[str, str]:
+ """Channel id -> name (cached through `ITeamsFiles.channels`)."""
+ if team_id not in self._channels:
+ try:
+ self._channels[team_id] = {c["id"]: c.get("displayName") or "" for c in self._teams_files.channels(self.note, self.broker, team_id)}
+ except Exception as ex:
+ GraphSearch.log.info("search: channels failed: %s", type(ex).__name__)
+ self._channels[team_id] = {}
+ return self._channels[team_id]
+
+ def shared_ids(self) -> set[str]:
+ if self._shared is None:
+ hit = self.cache.get(("search", "shared_ids"))
+ if hit is None:
+ try:
+ hit = {i["id"] for i in self._teams_files.shared_with_me(self.note, 200)}
+ except Exception as ex:
+ GraphSearch.log.info("search: sharedWithMe failed: %s", type(ex).__name__)
+ hit = set()
+ self.cache.set(("search", "shared_ids"), hit, GraphSearch.SHARED_TTL)
+ self._shared = hit
+ return self._shared
+
+
+class GraphSearch(IGraphSearch):
+ """The Search API over files, mail, chats and channel posts, with hit locations and a per-drive fallback. Never logs the query."""
+
+ log = logging.getLogger("schuly.microsoft")
+ SNIPPET_MAX = 300
+ MAP_TTL = 3600
+ SHARED_TTL = 600
+ FALLBACK_WORKERS = 4
+ FALLBACK_PER_DRIVE = 50
+ TEAM_SEARCH_CAP = 200
+ TAGS = re.compile(r"<[^>]*>")
+ TYPES = ("files", "mail", "chats", "channels")
+
+ def __init__(self, teams_files: ITeamsFiles):
+ self._teams_files = teams_files
+
+ def snippet(self, summary) -> str:
+ """The Search API's summary as plain text: `{inner}
' if style else f"{inner}
"})
+ if append and append.strip():
+ inner = self._html.sanitize_for_write(append)
+ if inner.strip():
+ commands.append({"target": "body", "action": "append", "content": f"{inner}
"})
+ if commands:
+ note.patch(f"{self.root(group)}/pages/{page_id}/content", commands, client=C_ONENOTE)
+ self.changed(broker)
+ return self.page(note, page_scoped)
+
+ def delete_page(self, note, broker, page_scoped: str):
+ group, page_id = self.parse_id(page_scoped)
+ note.delete(f"{self.root(group)}/pages/{page_id}", client=C_ONENOTE)
+ self.changed(broker)
+
+ def resource(self, broker, resource_scoped: str, limit: int) -> bytes:
+ """A picture's or a file's bytes, streamed from Graph's own host with the OneNote token and cut off past `limit`."""
+ group, resource_id = self.parse_id(resource_scoped)
+ url = f"{self.GRAPH}/{self.root(group)}/resources/{resource_id}/$value"
+ for attempt in range(self.RESOURCE_RETRIES + 1):
+ headers = {"Authorization": f"Bearer {broker(C_ONENOTE)}"}
+ # No redirects: the bearer token must never follow a response to another host.
+ with self._http.download_client().stream("GET", url, headers=headers, timeout=httpx.Timeout(60.0, connect=5.0), follow_redirects=False) as r:
+ if r.status_code in (429, 503) and attempt < self.RESOURCE_RETRIES:
+ wait = r.headers.get("Retry-After", "")
+ time.sleep(min(6.0, float(wait)) if wait.isdigit() else 1.0 + attempt)
+ continue
+ if r.status_code >= 300:
+ raise GraphError(r.status_code if r.status_code >= 400 else 502, "resource")
+ size = r.headers.get("Content-Length", "")
+ if size.isdigit() and int(size) > limit:
+ raise ResourceTooLarge()
+ data = bytearray()
+ for chunk in r.iter_bytes(64 * 1024):
+ data.extend(chunk)
+ if len(data) > limit:
+ raise ResourceTooLarge()
+ return bytes(data)
+ raise GraphError(429, "resource")
+
+ def _hit(self, group, p) -> dict:
+ return {"id": self.tag(group, p["id"]), "title": p.get("title") or "", "lastModifiedDateTime": p.get("lastModifiedDateTime"), "webUrl": self.web_url(p.get("links")), "section": self._ref(group, p.get("parentSection")), "notebook": self._ref(group, p.get("parentNotebook"))}
+
+ def _filtered(self, note, group, query: str, top: int) -> list[dict]:
+ literal = query.lower().replace("'", "''")
+ rows = note.list(f"{self.root(group)}/pages", {"$filter": f"contains(tolower(title),'{literal}')", "$select": "id,title,lastModifiedDateTime,links", "$expand": self.HIT_EXPAND, "$top": min(top, 100)}, limit=top, client=C_ONENOTE)
+ return [p for p in rows if self.GRAPH_ID.fullmatch(str(p.get("id", "")))]
+
+ def _filter_refused(self, ex: GraphError) -> bool:
+ """Graph's title filter covers a whole root and refuses one with too many sections, or with one section it cannot sync."""
+ return ex.status in (400, 422)
+
+ def _walk_search(self, note, broker, group, notebook_id, query: str, top: int) -> tuple[list[dict], bool]:
+ """Title search one section at a time (of one notebook, or of the whole root), for where Graph's own filter is refused."""
+ flat = [s for s in self._flat(note, broker, group, "sections") if notebook_id is None or (s.get("parentNotebook") or {}).get("id") == notebook_id]
+ flat.sort(key=lambda s: s.get("lastModifiedDateTime") or "", reverse=True)
+ partial = len(flat) > self.SEARCH_SECTIONS
+ needle = query.lower()
+
+ def titles(s):
+ try:
+ return self._kept(broker, ("onenote", "titles", group, s["id"]), lambda: note.list(f"{self.root(group)}/sections/{s['id']}/pages", {"$select": "id,title,lastModifiedDateTime,links", "$top": 100}, limit=500, client=C_ONENOTE))
+ except GraphError:
+ return None
+
+ # A few sections at a time, most recently changed first; their page titles are kept a short while for the next search.
+ with ThreadPoolExecutor(max_workers=self.TEAM_WORKERS) as pool:
+ found = list(pool.map(titles, flat[:self.SEARCH_SECTIONS]))
+ hits = []
+ for s, rows in zip(flat, found):
+ if rows is None:
+ # A section OneNote cannot sync is not one the person has pages in to find.
+ continue
+ parent = (s.get("parentNotebook") or {}).get("id")
+ for p in rows:
+ if needle in (p.get("title") or "").lower() and self.GRAPH_ID.fullmatch(str(p.get("id", ""))):
+ hits.append(self._hit(group, {**p, "parentSection": {"id": s["id"], "displayName": s.get("displayName")}, "parentNotebook": {"id": parent, "displayName": None} if parent else None}))
+ hits.sort(key=lambda h: h.get("lastModifiedDateTime") or "", reverse=True)
+ return hits[:top], partial
+
+ def _name_notebooks(self, note, broker, hits: list[dict]):
+ names = {n["id"]: n["displayName"] for n in self.notebooks(note, broker)}
+ for h in hits:
+ if h["notebook"] and not h["notebook"]["displayName"]:
+ h["notebook"]["displayName"] = names.get(h["notebook"]["id"], "")
+
+ def search(self, note, broker, query: str, notebook: str | None, top: int) -> dict:
+ query = re.sub(r"\s+", " ", query).strip()
+ if notebook:
+ group, notebook_id = self.parse_id(notebook)
+ try:
+ rows = [p for p in self._filtered(note, group, query, 100) if (p.get("parentNotebook") or {}).get("id") == notebook_id]
+ return {"items": [self._hit(group, p) for p in rows[:top]], "partial": False}
+ except GraphError as ex:
+ if not self._filter_refused(ex):
+ raise
+ hits, partial = self._walk_search(note, broker, group, notebook_id, query, top)
+ self._name_notebooks(note, broker, hits)
+ return {"items": hits, "partial": partial}
+ roots = [None] + [t["id"] for t in self._teams(note, broker)]
+
+ def one(group):
+ try:
+ return [self._hit(group, p) for p in self._filtered(note, group, query, top)], False
+ except GraphError as ex:
+ if not self._filter_refused(ex):
+ if group is None:
+ raise
+ return [], True
+ if group is None:
+ # The person's own notebooks are few sections, so they are walked; a class notebook is searched by picking it.
+ return self._walk_search(note, broker, None, None, query, top)
+ return [], True
+
+ with ThreadPoolExecutor(max_workers=self.TEAM_WORKERS) as pool:
+ found = list(pool.map(one, roots))
+ items = [h for rows, _ in found for h in rows]
+ self._name_notebooks(note, broker, items)
+ items.sort(key=lambda h: h.get("lastModifiedDateTime") or "", reverse=True)
+ return {"items": items[:top], "partial": any(p for _, p in found)}
diff --git a/src/infrastructure/graph/onenote_runner.py b/src/infrastructure/graph/onenote_runner.py
new file mode 100644
index 0000000..f4c98a8
--- /dev/null
+++ b/src/infrastructure/graph/onenote_runner.py
@@ -0,0 +1,27 @@
+from collections.abc import Callable
+
+from outnote import GraphError
+
+from src.application.abstractions.i_account_runner import IAccountRunner
+from src.application.abstractions.i_one_note_runner import IOneNoteRunner
+from src.domain.errors.service_error import ServiceError
+
+
+class OneNoteRunner(IOneNoteRunner):
+ """Graph's refusals for OneNote calls, mapped before the account runner sees them."""
+
+ def __init__(self, accounts: IAccountRunner):
+ self._accounts = accounts
+
+ async def run(self, refresh_token: str, action: Callable[..., object]) -> object:
+ def run(note, broker):
+ try:
+ return action(note, broker)
+ except GraphError as ex:
+ if ex.status == 403:
+ raise ServiceError(403, "OneNote refused access to this notebook.")
+ if ex.status in (400, 409, 422):
+ raise ServiceError(422, f"OneNote refused the request: {str(ex)[:200]}")
+ raise
+
+ return await self._accounts.run(refresh_token, run)
diff --git a/src/infrastructure/graph/pdf_converter.py b/src/infrastructure/graph/pdf_converter.py
new file mode 100644
index 0000000..075463f
--- /dev/null
+++ b/src/infrastructure/graph/pdf_converter.py
@@ -0,0 +1,155 @@
+"""Office files as PDF for the web viewer, converted by Microsoft 365 (Graph's `content?format=pdf`).
+
+A file already in OneDrive / SharePoint (a drive item, a file shared in a chat) is converted where it is. A file that isn't in a
+drive (a mail attachment, bytes the caller uploads) goes up to a temporary file in the app's own OneDrive folder
+(`special/approot:/schuly-preview/{inner}
' if style else f"{inner}
"})
- if append and append.strip():
- inner = onenote_html.sanitize_for_write(append)
- if inner.strip():
- commands.append({"target": "body", "action": "append", "content": f"{inner}
"})
- if commands:
- note.patch(f"{root(group)}/pages/{page_id}/content", commands, client=C_ONENOTE)
- changed(broker)
- return page(note, page_scoped)
-
-
-def delete_page(note, broker, page_scoped: str):
- group, page_id = parse_id(page_scoped)
- note.delete(f"{root(group)}/pages/{page_id}", client=C_ONENOTE)
- changed(broker)
-
-
-# --- resources -------------------------------------------------------------------------------
-
-
-def resource(broker, resource_scoped: str, limit: int) -> bytes:
- """A picture's or a file's bytes, streamed from Graph's own host with the OneNote token and cut off past `limit`."""
- group, resource_id = parse_id(resource_scoped)
- url = f"{GRAPH}/{root(group)}/resources/{resource_id}/$value"
- for attempt in range(RESOURCE_RETRIES + 1):
- headers = {"Authorization": f"Bearer {broker(C_ONENOTE)}"}
- # No redirects: the bearer token must never follow a response to another host.
- with outnote_client.download_client().stream("GET", url, headers=headers, timeout=httpx.Timeout(60.0, connect=5.0), follow_redirects=False) as r:
- if r.status_code in (429, 503) and attempt < RESOURCE_RETRIES:
- wait = r.headers.get("Retry-After", "")
- time.sleep(min(6.0, float(wait)) if wait.isdigit() else 1.0 + attempt)
- continue
- if r.status_code >= 300:
- raise GraphError(r.status_code if r.status_code >= 400 else 502, "resource")
- size = r.headers.get("Content-Length", "")
- if size.isdigit() and int(size) > limit:
- raise ResourceTooLarge()
- data = bytearray()
- for chunk in r.iter_bytes(64 * 1024):
- data.extend(chunk)
- if len(data) > limit:
- raise ResourceTooLarge()
- return bytes(data)
- raise GraphError(429, "resource")
-
-
-# --- search ----------------------------------------------------------------------------------
-
-
-def _hit(group, p) -> dict:
- return {"id": tag(group, p["id"]), "title": p.get("title") or "", "lastModifiedDateTime": p.get("lastModifiedDateTime"), "webUrl": web_url(p.get("links")), "section": _ref(group, p.get("parentSection")), "notebook": _ref(group, p.get("parentNotebook"))}
-
-
-def _filtered(note, group, query: str, top: int) -> list[dict]:
- literal = query.lower().replace("'", "''")
- rows = note.list(f"{root(group)}/pages", {"$filter": f"contains(tolower(title),'{literal}')", "$select": "id,title,lastModifiedDateTime,links", "$expand": HIT_EXPAND, "$top": min(top, 100)}, limit=top, client=C_ONENOTE)
- return [p for p in rows if GRAPH_ID.fullmatch(str(p.get("id", "")))]
-
-
-def _filter_refused(ex: GraphError) -> bool:
- """Graph's title filter covers a whole root and refuses one with too many sections, or with one section it cannot sync."""
- return ex.status in (400, 422)
-
-
-def _walk_search(note, broker, group, notebook_id, query: str, top: int) -> tuple[list[dict], bool]:
- """Title search one section at a time (of one notebook, or of the whole root), for where Graph's own filter is refused."""
- flat = [s for s in _flat(note, broker, group, "sections") if notebook_id is None or (s.get("parentNotebook") or {}).get("id") == notebook_id]
- flat.sort(key=lambda s: s.get("lastModifiedDateTime") or "", reverse=True)
- partial = len(flat) > SEARCH_SECTIONS
- needle = query.lower()
-
- def titles(s):
- try:
- return _kept(broker, ("onenote", "titles", group, s["id"]), lambda: note.list(f"{root(group)}/sections/{s['id']}/pages", {"$select": "id,title,lastModifiedDateTime,links", "$top": 100}, limit=500, client=C_ONENOTE))
- except GraphError:
- return None
-
- # A few sections at a time, most recently changed first; their page titles are kept a short while for the next search.
- with ThreadPoolExecutor(max_workers=TEAM_WORKERS) as pool:
- found = list(pool.map(titles, flat[:SEARCH_SECTIONS]))
- hits = []
- for s, rows in zip(flat, found):
- if rows is None:
- # A section OneNote cannot sync is not one the person has pages in to find.
- continue
- parent = (s.get("parentNotebook") or {}).get("id")
- for p in rows:
- if needle in (p.get("title") or "").lower() and GRAPH_ID.fullmatch(str(p.get("id", ""))):
- hits.append(_hit(group, {**p, "parentSection": {"id": s["id"], "displayName": s.get("displayName")}, "parentNotebook": {"id": parent, "displayName": None} if parent else None}))
- hits.sort(key=lambda h: h.get("lastModifiedDateTime") or "", reverse=True)
- return hits[:top], partial
-
-
-def _name_notebooks(note, broker, hits: list[dict]):
- names = {n["id"]: n["displayName"] for n in notebooks(note, broker)}
- for h in hits:
- if h["notebook"] and not h["notebook"]["displayName"]:
- h["notebook"]["displayName"] = names.get(h["notebook"]["id"], "")
-
-
-def search(note, broker, query: str, notebook: str | None, top: int) -> dict:
- query = re.sub(r"\s+", " ", query).strip()
- if notebook:
- group, notebook_id = parse_id(notebook)
- try:
- rows = [p for p in _filtered(note, group, query, 100) if (p.get("parentNotebook") or {}).get("id") == notebook_id]
- return {"items": [_hit(group, p) for p in rows[:top]], "partial": False}
- except GraphError as ex:
- if not _filter_refused(ex):
- raise
- hits, partial = _walk_search(note, broker, group, notebook_id, query, top)
- _name_notebooks(note, broker, hits)
- return {"items": hits, "partial": partial}
- roots = [None] + [t["id"] for t in _teams(note, broker)]
-
- def one(group):
- try:
- return [_hit(group, p) for p in _filtered(note, group, query, top)], False
- except GraphError as ex:
- if not _filter_refused(ex):
- if group is None:
- raise
- return [], True
- if group is None:
- # The person's own notebooks are few sections, so they are walked; a class notebook is searched by picking it.
- return _walk_search(note, broker, None, None, query, top)
- return [], True
-
- with ThreadPoolExecutor(max_workers=TEAM_WORKERS) as pool:
- found = list(pool.map(one, roots))
- items = [h for rows, _ in found for h in rows]
- _name_notebooks(note, broker, items)
- items.sort(key=lambda h: h.get("lastModifiedDateTime") or "", reverse=True)
- return {"items": items[:top], "partial": any(p for _, p in found)}
diff --git a/src/onenote_html.py b/src/onenote_html.py
deleted file mode 100644
index 08bb672..0000000
--- a/src/onenote_html.py
+++ /dev/null
@@ -1,487 +0,0 @@
-"""OneNote page HTML, in both directions, and the page's ink.
-
-Reading is the sanitising step. A page is parsed into a small tree and rebuilt from an allowlist:
-nothing of the page's own markup is passed through. Scripts, styles, event attributes, forms and
-every tag outside the list are dropped (the text inside an unknown tag is kept), styles keep only a
-few text properties with plain values, links keep only http, https and mailto, and positioned items
-(OneNote's text boxes, pictures and files) keep only a numeric left, top and width.
-
-Pictures and attached files are OneNote resources behind the account's token. Their address is
-never handed out: an `img` keeps only `data-resource` (the resource id, scoped like every other id)
-and the caller fetches the bytes through the resource route. Ink, which Graph only sends as InkML
-in a second multipart part, becomes inline SVG path data.
-
-Writing (a new page, a replaced text box, appended text) goes through the same allowlist with no
-styles beyond simple text formatting, and comes out as the XHTML OneNote takes.
-
-A text box is editable only when it holds nothing but text: replacing a box with a picture, a file
-or ink in it would drop them, so such a box is shown and never written over.
-"""
-
-import html
-import re
-from html.parser import HTMLParser
-from urllib.parse import urlsplit
-
-TEXT_TAGS = {"p", "div", "span", "b", "strong", "i", "em", "u", "s", "strike", "del", "sub", "sup", "a", "br", "ul", "ol", "li", "h1", "h2", "h3", "h4", "h5", "h6", "table", "thead", "tbody", "tfoot", "tr", "td", "th", "pre", "code", "blockquote", "cite"}
-VOID = {"br", "img", "hr", "meta", "link", "input", "col", "area", "base", "wbr", "source", "param", "embed"}
-DROP_CONTENT = {"script", "style", "head", "title", "template", "noscript", "iframe", "frame", "frameset", "svg", "math", "canvas", "video", "audio", "form", "button", "select", "textarea", "applet", "embed", "xml"}
-LINK_SCHEMES = ("http", "https", "mailto")
-STYLE_PROPS = {"color", "background-color", "font-family", "font-size", "font-weight", "font-style", "text-decoration", "text-align", "vertical-align", "margin-top", "margin-bottom", "margin-left", "line-height", "border", "border-width", "border-style", "border-color", "border-collapse", "padding", "width"}
-WRITE_STYLE_PROPS = {"color", "background-color", "font-weight", "font-style", "text-decoration", "text-align"}
-STYLE_VALUE = re.compile(r"[#\w\s.,%'\"()+-]{1,100}")
-STYLE_BLOCKED = ("url", "expression", "javascript", "image", "attr(", "var(", "env(")
-NUMBER = re.compile(r"-?\d{1,6}(?:\.\d{1,3})?")
-TAG_VALUE = re.compile(r"[a-z0-9:,-]{1,60}")
-CONTENT_TYPE = re.compile(r"[a-z0-9][\w.+-]{0,126}/[a-z0-9][\w.+-]{0,126}")
-COLOR = re.compile(r"#[0-9A-Fa-f]{6}")
-# The ids OneNote gives page elements (`div:{guid}{n}`) and its resources (`0-abc!1-def`).
-SAFE_ID = re.compile(r"[A-Za-z0-9!_{}:.\-]{1,200}")
-RESOURCE_PATH = re.compile(r"/onenote/resources/([A-Za-z0-9!_{}:.\-]{1,200})/(?:\$value|content)$")
-GRAPH_HOST = "graph.microsoft.com"
-MAX_ALT = 2000
-MAX_NAME = 255
-# Ink bounds: a page drawn on for a whole lesson is thousands of strokes; past these the rest is left out.
-MAX_TRACES = 5000
-MAX_POINTS = 400_000
-PX_PER_HIMETRIC = 96.0 / 2540.0
-BOX_WIDTH = 600
-BOX_HEIGHT = 120
-
-
-class _Node:
- __slots__ = ("tag", "attrs", "children")
-
- def __init__(self, tag: str, attrs: dict[str, str]):
- self.tag = tag
- self.attrs = attrs
- self.children: list = []
-
-
-class _Tree(HTMLParser):
- """A forgiving tree builder: OneNote's markup is regular, and anything that is not is closed where it can be."""
-
- def __init__(self):
- super().__init__(convert_charrefs=True)
- self.root = _Node("#root", {})
- self.stack = [self.root]
-
- def handle_starttag(self, tag, attrs):
- node = _Node(tag.lower(), {k.lower(): (v or "") for k, v in attrs})
- self.stack[-1].children.append(node)
- if node.tag not in VOID:
- self.stack.append(node)
-
- def handle_startendtag(self, tag, attrs):
- self.stack[-1].children.append(_Node(tag.lower(), {k.lower(): (v or "") for k, v in attrs}))
-
- def handle_endtag(self, tag):
- tag = tag.lower()
- for i in range(len(self.stack) - 1, 0, -1):
- if self.stack[i].tag == tag:
- del self.stack[i:]
- return
-
- def handle_data(self, data):
- self.stack[-1].children.append(data)
-
-
-def _parse(markup: str) -> _Node:
- tree = _Tree()
- tree.feed(markup)
- tree.close()
- return tree.root
-
-
-def _find(node: _Node, tag: str) -> _Node | None:
- for child in node.children:
- if isinstance(child, _Node):
- if child.tag == tag:
- return child
- found = _find(child, tag)
- if found is not None:
- return found
- return None
-
-
-def _text(node: _Node) -> str:
- return "".join(c if isinstance(c, str) else _text(c) for c in node.children)
-
-
-def safe_href(value: str | None) -> str | None:
- value = (value or "").strip()
- if not value or len(value) > 2000 or any(ord(ch) < 32 for ch in value):
- return None
- try:
- parts = urlsplit(value)
- except ValueError:
- return None
- scheme = parts.scheme.lower()
- if scheme not in LINK_SCHEMES or (scheme != "mailto" and not parts.netloc):
- return None
- return value
-
-
-def _style(value: str | None, allowed: set[str]) -> str:
- out = []
- for rule in (value or "").split(";"):
- name, sep, val = rule.partition(":")
- name, val = name.strip().lower(), val.strip()
- if not sep or name not in allowed or not STYLE_VALUE.fullmatch(val) or any(b in val.lower() for b in STYLE_BLOCKED):
- continue
- out.append(f"{name}:{val}")
- return ";".join(out)
-
-
-def _length(style: str | None, prop: str) -> float | None:
- """A number of CSS px out of `left:48px;top:12pt` (a pt value is scaled)."""
- for rule in (style or "").split(";"):
- name, sep, val = rule.partition(":")
- if sep and name.strip().lower() == prop:
- val = val.strip().lower()
- unit = "pt" if val.endswith("pt") else ""
- number = val.removesuffix("px").removesuffix("pt").strip()
- if NUMBER.fullmatch(number):
- px = float(number) * (96.0 / 72.0 if unit == "pt" else 1.0)
- return round(px, 1)
- return None
-
-
-def _attr_number(value: str | None) -> float | None:
- value = (value or "").strip().removesuffix("px")
- return round(float(value), 1) if NUMBER.fullmatch(value) else None
-
-
-def _num(value: float) -> str:
- return str(int(value)) if value == int(value) else str(value)
-
-
-def scoped(scope: str | None, graph_id: str) -> str:
- """An id handed out: bare for the person's own OneNote, `" if self.write else "
" - attrs = [] - if tag == "a": - href = safe_href(node.attrs.get("href")) - if href: - attrs.append(f'href="{html.escape(href, quote=True)}"') - if not self.write: - attrs.append('rel="noopener noreferrer"') - if tag in ("td", "th"): - for name in ("colspan", "rowspan"): - value = node.attrs.get(name, "") - if value.isdigit() and 1 <= int(value) <= 100: - attrs.append(f'{name}="{int(value)}"') - tag_value = node.attrs.get("data-tag", "").lower() - if tag_value and TAG_VALUE.fullmatch(tag_value) and tag in ("p", "li", "span", "div", "h1", "h2", "h3", "h4", "h5", "h6"): - attrs.append(f'data-tag="{tag_value}"') - style = _style(node.attrs.get("style"), WRITE_STYLE_PROPS if self.write else STYLE_PROPS) - if style: - attrs.append(f'style="{html.escape(style, quote=True)}"') - head = f"<{tag} {' '.join(attrs)}>" if attrs else f"<{tag}>" - return f"{head}{self.children(node)}" - - def _position(self, node: _Node, positioned: bool, width: float | None = None) -> str: - if not positioned: - return "" - style = node.attrs.get("style") - left, top = _length(style, "left") or 0.0, _length(style, "top") or 0.0 - parts = [f"position:absolute;left:{_num(left)}px;top:{_num(top)}px"] - if width is not None: - parts.append(f"width:{_num(width)}px") - return f' style="{";".join(parts)}"' - - def image(self, node: _Node, positioned: bool) -> str: - rid = resource_id(node.attrs.get("data-fullres-src")) or resource_id(node.attrs.get("src")) - if rid is None: - return "" - sid = scoped(self.scope, rid) - kind = (node.attrs.get("data-fullres-src-type") or node.attrs.get("data-src-type") or "").lower() - self.resources.setdefault(sid, {"id": sid, "kind": "image", "name": None, "contentType": kind if CONTENT_TYPE.fullmatch(kind) else None}) - alt = (node.attrs.get("alt") or "")[:MAX_ALT] - width, height = _attr_number(node.attrs.get("width")), _attr_number(node.attrs.get("height")) - size = (f' width="{_num(width)}"' if width else "") + (f' height="{_num(height)}"' if height else "") - return f'
{sanitize_for_write(body)}
" if body and body.strip() else ""
- return f"go
go
go