Add photo upload, face framing and derived avatars (JVNAUTOSCI-2749)

docs/engineering/unified_conversations_and_participant_profiles.md

@@ -91,8 +91,28 @@ Avatar variants use Von's existing scope modes:
 Canonical file-copy visibility enforces the audience. Matching combined, user,
 organisation and global variants take precedence in that order; a variant from a
 different organisation is not selected as the current organisation's portrait.
-Publication creates a centred 256-pixel PNG derivative with source metadata
-removed. The original upload/generated candidate stays private. Replacing or
+Publication creates a 256-pixel PNG derivative with source metadata removed.
+The editor accepts a file picker or one dropped PNG, JPEG or WebP (up to 8 MiB),
+preserves the source through the private image store, and offers zoom and position
+controls. `/avatar/prepare` applies EXIF orientation before suggesting framing.
+The bundled OpenCV frontal-face detector runs locally: one detected face gets a
+suggested crop, while zero or multiple faces retain a centred crop for manual
+correction. Detector failure also preserves manual upload; locations do not
+identify people or enrol biometric identities. Detection can miss faces and is
+only an adjustable default.
+
+`/avatar` accepts an optional square `crop` (`x`, `y`, `size`) in oriented source
+pixels. `/avatar/generate` accepts a private `source_image_concept_id`; after
+checking its owner, it sends oriented, metadata-free photo pixels to the existing
+configured OpenAI client using image edits with `gpt-image-1.5`. The style is the
+user's prompt. Without a source it retains text-only generation. These are
+distinct provider operations; image-edit failure does not fall back to text-only
+generation. Generated candidates record their private source provenance and
+require explicit **Use avatar**. **Return to source photo** restores manual
+framing, including after provider failure. Existing MCP calls retain their
+previous text-generation and centred-publication interface.
+
+The original upload/generated candidate stays private. Replacing or
 removing a variant retires that publication, allowing another applicable variant
 to become visible. Image reads validate the viewer and window context, use private
 no-store caching, and never expose the private source blob URL in the public

pyproject.toml

@@ -81,6 +81,7 @@ dependencies = [
     "zope.proxy==5.2",
     "customtkinter>=5.2.2",
     "pillow>=8.0.0",
+    "opencv-python-headless>=4.10,<5",
     "annotated-types>=0.7.0",
     "asttokens>=3.0.0",
     "bibtexparser>=1.4.3",

src/backend/server/routes/participant_profile_routes.py

@@ -1,6 +1,7 @@
 """Common profile API for Settings, concepts, conversations and agent tools."""
 
 import io
+import json
 import logging
 
 from flask import jsonify, request, send_file, session
@@ -19,12 +20,16 @@ def perform(action):
             return jsonify(error="profile_unavailable", message=str(exc)), 403
         except ValueError as exc:
             return jsonify(error="invalid_profile_image", message=str(exc)), 400
-        except Exception:  # noqa: BLE001 - provider/storage failures are reported at the HTTP boundary
+        # Provider/storage failures are reported at the HTTP boundary.
+        except Exception:  # noqa: BLE001
             _log.exception("Participant profile operation failed")
-            return jsonify(
-                error="profile_operation_failed",
-                message="The profile operation failed. Your existing avatar is unchanged unless saving completed; refresh to check.",
-            ), 503
+            return (
+                jsonify(
+                    error="profile_operation_failed",
+                    message="The profile operation failed. Your existing avatar is unchanged unless saving completed; refresh to check.",
+                ),
+                503,
+            )
 
     @blueprint.route("/api/participants/profile", methods=["GET"])
     def participant_profile():
@@ -54,10 +59,29 @@ def participant_avatar_save():
                     remove=payload.get("remove") is True,
                     scope=payload.get("scope", "global_general"),
                     data=uploaded.read(MAX_IMAGE_BYTES + 1) if uploaded else None,
+                    crop=(
+                        json.loads(payload["crop"])
+                        if isinstance(payload.get("crop"), str)
+                        else payload.get("crop")
+                    ),
                 )
             }
         )
 
+    @blueprint.route("/api/participants/avatar/prepare", methods=["POST"])
+    def participant_avatar_prepare():
+        def prepare():
+            uploaded = request.files.get("file")
+            if uploaded is None:
+                raise ValueError("Choose an image first.")
+            return profiles.prepare_avatar(
+                data=uploaded.read(MAX_IMAGE_BYTES + 1),
+                filename=uploaded.filename or "avatar-source.png",
+                concept_id=request.form.get("concept_id"),
+            )
+
+        return perform(prepare)
+
     @blueprint.route("/api/participants/avatar/generate", methods=["POST"])
     def participant_avatar_generate():
         payload = request.get_json(silent=True) or {}
@@ -66,6 +90,7 @@ def participant_avatar_generate():
                 "image": profiles.generate_avatar(
                     concept_id=payload.get("concept_id"),
                     prompt=payload.get("prompt"),
+                    source_image_concept_id=payload.get("source_image_concept_id"),
                 )
             }
         )

src/backend/services/avatar_framing.py

@@ -0,0 +1,29 @@
+"""Local face locations for an adjustable avatar crop; no identity inference."""
+
+from PIL import Image
+
+
+def find_faces(image: Image.Image) -> list[dict]:
+    import cv2
+    import numpy as np
+
+    # OpenCV ships this detector. No remote model or photo transfer is needed.
+    detector = cv2.CascadeClassifier(
+        cv2.data.haarcascades + "haarcascade_frontalface_default.xml"
+    )
+    thumbnail = image.copy()
+    thumbnail.thumbnail((1024, 1024))
+    grey = cv2.cvtColor(np.asarray(thumbnail), cv2.COLOR_RGB2GRAY)
+    boxes = detector.detectMultiScale(
+        grey, scaleFactor=1.1, minNeighbors=5, minSize=(24, 24)
+    )
+    sx, sy = image.width / thumbnail.width, image.height / thumbnail.height
+    return [
+        {
+            "x": float(x * sx),
+            "y": float(y * sy),
+            "width": float(w * sx),
+            "height": float(h * sy),
+        }
+        for x, y, w, h in boxes
+    ]

src/backend/services/participant_profile_service.py

@@ -9,6 +9,8 @@
 import base64
 import hashlib
 import io
+import logging
+import math
 from urllib.parse import quote
 
 from PIL import Image, ImageOps
@@ -137,13 +139,17 @@ def get_profiles(concept_ids: list[str]) -> list[dict]:
                 or cid.removeprefix("#V#").replace("_", " "),
                 "can_edit": actor == cid and _can_edit_profile(actor),
                 "avatar_scope": avatar.get("avatar_scope"),
-                "avatar_url": f"/von/api/participants/{quote(cid, safe='')}/avatar?v={avatar['sha256']}"
-                if avatar.get("sha256")
-                else None,
+                "avatar_url": (
+                    f"/von/api/participants/{quote(cid, safe='')}/avatar?v={avatar['sha256']}"
+                    if avatar.get("sha256")
+                    else None
+                ),
                 "avatar_version": avatar.get("sha256"),
-                "available_scopes": list(SCOPES)
-                if get_effective_organisation_concept_id()
-                else ["user_only_default", "global_general"],
+                "available_scopes": (
+                    list(SCOPES)
+                    if get_effective_organisation_concept_id()
+                    else ["user_only_default", "global_general"]
+                ),
             }
         )
     return result
@@ -164,6 +170,7 @@ def set_avatar(
     data: bytes | None = None,
     remove: bool = False,
     scope: str = "global_general",
+    crop: dict | None = None,
 ) -> dict:
     actor, concept_id, _ = _participant(concept_id, edit=True)
     from .concept_service import update_concept
@@ -193,6 +200,8 @@ def set_avatar(
     inspect_image(data)
     with Image.open(io.BytesIO(data)) as original:
         oriented = ImageOps.exif_transpose(original)
+        if crop is not None:
+            oriented = oriented.crop(_crop_box(crop, *oriented.size))
         # Reconstruct pixels so EXIF, comments and private source metadata cannot
         # be carried into the published derivative.
         resized = ImageOps.fit(
@@ -254,25 +263,91 @@ def load_avatar(concept_id: str) -> bytes:
     return data
 
 
-def generate_avatar(*, prompt: str, concept_id: str | None = None) -> dict:
+def _crop_box(crop: dict, width: int, height: int) -> tuple:
+    if not isinstance(crop, dict) or set(crop) != {"x", "y", "size"}:
+        raise ValueError("Supply a square crop with x, y and size.")
+    x, y, size = (crop[k] for k in ("x", "y", "size"))
+    if any(type(v) not in (int, float) or not math.isfinite(v) for v in (x, y, size)):
+        raise ValueError("Crop coordinates must be finite numbers.")
+    if x < 0 or y < 0 or size < 1 or x + size > width or y + size > height:
+        raise ValueError("Crop must lie within the oriented source image.")
+    return (x, y, x + size, y + size)
+
+
+def prepare_avatar(
+    *, data: bytes, filename: str, concept_id: str | None = None
+) -> dict:
+    """Preserve the private original and suggest framing without identifying anyone."""
+    actor, _, _ = _participant(concept_id, edit=True)
+    inspect_image(data)
+    with Image.open(io.BytesIO(data)) as original:
+        oriented = ImageOps.exif_transpose(original).convert("RGB")
+    width, height = oriented.size
+    from .avatar_framing import find_faces
+
+    detection_available = True
+    try:
+        faces = find_faces(oriented)
+    # A detector failure must not prevent ordinary uploads and manual cropping.
+    except Exception:  # noqa: BLE001
+        logging.getLogger(__name__).warning("Avatar face detection unavailable")
+        detection_available, faces = False, []
+    size = min(width, height)
+    x, y = (width - size) / 2, (height - size) / 2
+    if len(faces) == 1:
+        face = faces[0]
+        size = min(size, max(face["width"], face["height"]) * 2)
+        x = min(width - size, max(0, face["x"] + face["width"] / 2 - size / 2))
+        y = min(height - size, max(0, face["y"] + face["height"] / 2 - size * 0.45))
+    image = store_image(data=data, filename=filename, user_concept_id=actor)
+    return {
+        "image": image,
+        "width": width,
+        "height": height,
+        "face_count": len(faces),
+        "face_detection_available": detection_available,
+        "crop": {"x": x, "y": y, "size": size},
+    }
+
+
+def generate_avatar(
+    *,
+    prompt: str,
+    concept_id: str | None = None,
+    source_image_concept_id: str | None = None,
+) -> dict:
     """Generate a private candidate; publishing is a separate, reusable action."""
     actor, _, _ = _participant(concept_id, edit=True)
     if not isinstance(prompt, str) or not prompt.strip() or len(prompt) > 4000:
         raise ValueError("Describe the image in 1–4000 characters.")
     from ..languagemodels.openai_client import OpenAIClient
 
+    source = None
+    if source_image_concept_id:
+        source, original_data = load_image(source_image_concept_id, actor)
+        # Send oriented pixels, without private EXIF, to the configured provider.
+        with Image.open(io.BytesIO(original_data)) as original:
+            oriented = ImageOps.exif_transpose(original).convert("RGB")
+            clean = Image.new("RGB", oriented.size)
+            clean.paste(oriented)
+            image_input = io.BytesIO()
+            clean.save(image_input, format="PNG")
+        image_input.name = "avatar-source.png"
+        image_input.seek(0)
     # Bounded single image; no automatic retry that could duplicate provider cost.
+    images = OpenAIClient().client.with_options(max_retries=0).images
+    options = {
+        "model": "gpt-image-1.5",
+        "prompt": prompt.strip(),
+        "n": 1,
+        "size": "1024x1024",
+        "quality": "low",
+        "output_format": "png",
+    }
     result = (
-        OpenAIClient()
-        .client.with_options(max_retries=0)
-        .images.generate(
-            model="gpt-image-1.5",
-            prompt=prompt.strip(),
-            n=1,
-            size="1024x1024",
-            quality="low",
-            output_format="png",
-        )
+        images.edit(image=image_input, **options)
+        if source
+        else images.generate(**options)
     )
     if not result.data or not result.data[0].b64_json:
         raise ValueError("The image provider returned no image.")
@@ -285,5 +360,13 @@ def generate_avatar(*, prompt: str, concept_id: str | None = None) -> dict:
             "kind": "generated",
             "model": "gpt-image-1.5",
             "prompt": prompt.strip(),
+            **(
+                {
+                    "parent_concept_id": source_image_concept_id,
+                    "parent_sha256": source["sha256"],
+                }
+                if source
+                else {}
+            ),
         },
     )