From f6c327cb83762963a2fc8d33c9018805494349eb Mon Sep 17 00:00:00 2001 From: witbrock Date: Thu, 10 Sep 2026 13:45:03 -0400 Subject: [PATCH 1/2] feat(tasks): retain Jira originals and support project cutover (JVNAUTOSCI-2737) --- AGENTS.md | 7 + ...t_jira_to_von_migration_jvnautosci_1223.md | 137 +- scripts/backup_task_project_files.py | 226 ++++ scripts/backup_von_db.py | 3 +- scripts/capture_jira_project_sources.py | 209 +++ scripts/restore_von_db.py | 23 +- scripts/run_jira_task_migration.py | 43 +- .../integrations/internal_mcp/catalogue.py | 285 +++- .../internal_mcp/jira_migration_resources.py | 133 ++ .../internal_mcp/jira_proxy_mcp.py | 54 +- .../internal_mcp/tool_contract_registry.py | 9 + src/backend/mcp_server/mcp_server.py | 65 + src/backend/mcp_server/mcp_stdio_server.py | 125 +- src/backend/mcp_server/vontology_mcp.json | 1148 +++++++++++++++++ src/backend/server/routes/task_routes.py | 133 +- .../services/computer_file_copy_service.py | 35 +- .../services/jira_source_retention_service.py | 609 +++++++++ .../services/jira_task_import_service.py | 486 ++++++- .../jira_task_migration_runner_service.py | 131 +- .../jira_task_reconciliation_service.py | 289 +++++ .../services/task_management_service.py | 241 +++- src/backend/services/task_project_service.py | 365 ++++++ src/backend/services/tool_metadata_service.py | 20 + .../static/js/components/taskPanel.js | 187 ++- .../test_internal_mcp_task_parity_tools.py | 2 + tests/backend/test_jira_cutover_integrity.py | 362 ++++++ tests/backend/test_jira_read_authority.py | 11 +- .../test_jira_source_retention_service.py | 182 +++ .../backend/test_jira_task_import_service.py | 170 +++ ...test_jira_task_migration_runner_service.py | 72 ++ ...t_mcp_stdio_coding_agent_access_profile.py | 16 +- tests/backend/test_mcp_stdio_task_actor.py | 141 ++ tests/backend/test_task_project_service.py | 80 ++ tests/frontend/taskPanelGroupFilters.test.js | 56 + tests/test_backup_task_project_files.py | 89 ++ tests/test_restore_von_db.py | 24 + 36 files changed, 6038 insertions(+), 130 deletions(-) create mode 100644 scripts/backup_task_project_files.py create mode 100644 scripts/capture_jira_project_sources.py create mode 100644 src/backend/integrations/internal_mcp/jira_migration_resources.py create mode 100644 src/backend/services/jira_source_retention_service.py create mode 100644 src/backend/services/task_project_service.py create mode 100644 tests/backend/test_jira_cutover_integrity.py create mode 100644 tests/backend/test_jira_source_retention_service.py create mode 100644 tests/backend/test_mcp_stdio_task_actor.py create mode 100644 tests/backend/test_task_project_service.py create mode 100644 tests/test_backup_task_project_files.py diff --git a/AGENTS.md b/AGENTS.md index f19edac27..c8caa8943 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -407,6 +407,13 @@ mechanism, or grant of merge authority. ## 8. Task and repository discipline +- For a project represented in Von, read its task-project `writer` before + choosing the tracking surface. When it is `von`, use native task tools and + the associated collections for creation, updates and completion; legacy Jira + keys remain lookup aliases. When it is `jira`, keep Jira authoritative and + treat imported tasks as migration copies. Keep Jira access available for + provenance and migration recovery. See the + [Jira migration guide](docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md). - Use New Zealand English spelling by default. - Use the native shell for the host: PowerShell on Windows; `zsh`/`sh` on macOS and Linux, except when deliberately invoking or testing another shell. diff --git a/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md b/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md index f008a5fdd..ad47ca429 100644 --- a/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md +++ b/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md @@ -1,6 +1,137 @@ -# JVNAUTOSCI-1223 Workflow-First Jira -> Von Migration - -Date: 2026-02-21 +# Jira to Von task migration and project cutover + +Last updated: 10 September 2026. Current delivery decision: +[JVNAUTOSCI-2737](https://naoinstitute.atlassian.net/browse/JVNAUTOSCI-2737). + +## Full source retention and project cutover + +The ordinary task projection and the retained Jira original serve different +purposes. A task count or a successful import does not establish full content +retention. `--preserve-source` keeps complete source documents, rendered fields, +field schemas/names, paginated comments, worklogs, history, properties, remote +links, votes/watchers, and original attachment bytes in immutable, hashed Von +file copies. Native task fields support day-to-day work; unsupported fields +remain retrievable in the original. Capture errors remain explicit and block a +full retention claim. + +Each source project has a stable Von project record and an associated primary +task collection. Board and saved-filter snapshots can be associated with every +project they cover. Collections preserve their source definitions and selected +issue keys; native additions use explicit membership. They do not execute JQL +or reproduce Jira board automation. Project descriptions, properties, roles, +components, versions, workflow/status and security configuration are retained +as source information. Source sharing settings do not grant access in Von. + +### Run and recover an import + +Use the canonical runner with an explicitly configured local operator: + +```sh +pdm run python scripts/run_jira_task_migration.py \ + --project-key PROJECT --actor-concept-id '#V#operator' \ + --preserve-source --include-done --passes 1 --batch-size 25 \ + --resume --report-path /private/audit/PROJECT.json +``` + +Supply `--organisation-concept-id` only for the intended organisational project. +Other projects can remain private to the operator. Source project IDs and site +identity distinguish projects; stable issue IDs and previous keys preserve +lookup after source moves. All issue types and terminal statuses are included +by this command. Do not use `--only-missing` when existing projections still +need original-content retention. + +For attachments above the Jira proxy's default 10 MiB retrieval limit, set +`VON_INTERNAL_MCP_JIRA_ATTACHMENT_MAX_SIZE_BYTES=104857600` in the migration +process environment. This explicitly permits retrieval up to 100 MiB; larger +files remain reported gaps requiring a separate bounded capture. + +Completed batches are checkpointed atomically. Repeating the command resumes a +matching source snapshot. If Jira has changed, use a fresh report path for the +delta and reconcile it against fresh full discovery. Reruns retain native edits +using the previous import projection, report conflicts, preserve prior source +archives, and deduplicate source activity. Unresolved hierarchy/link targets +can be repaired from retained import rows after all projects have imported. +Never infer deletion authority from a missing source issue. + +`scripts/capture_jira_project_sources.py` captures project and site source +configuration through named Jira MCP resources and associates board/filter +collections. Inspect its report for unsupported or inaccessible resources. +Automation exports, cloud exports, app-owned data, forms and linked external +documents require explicit capture or an evidence-backed scope disposition; +REST access failures are not proof that a resource is empty. Keep these private +source artefacts outside the repository. + +### Native use and actor binding + +The Tasks panel supports project/collection selection, ordinary task edits, +hierarchy, comments, attachments, worklogs and transitions. Imported tasks link +to their retained original without needing Jira. Native MCP tools expose the +same canonical services, including `task_list_projects`, `task_get_project`, +`task_get_collection`, `task_get`, `task_search` and `task_get_source_archive`. +`task_get` accepts legacy Jira keys as well as Von task concept IDs. + +A local stdio connection can bind the trusted operator once in its server +environment: + +```toml +[mcp_servers.vontology.env] +VON_MCP_TASK_ACTOR_CONCEPT_ID = "#V#operator" +VON_MCP_TASK_ORGANISATION_CONCEPT_ID = "#V#organisation" +``` + +The organisation is optional. This is local operator configuration, not a +client-supplied identity claim. It applies only to task tools, retains canonical +access checks and the configured write profile, rejects a conflicting actor +payload, and never replaces an inherited actor context. Browser and internal +workflow calls continue to derive identity from their trusted session context. + +### Reconciliation, backup and writer selection + +`reconcile_retained_project` compares fresh source IDs, keys, types and versions +with task membership, source relations, project originals and attachment hashes. +Keep its missing counts and discrepancy list alongside the import report. A +successful result proves the checked retention scope; it does not itself prove +the project is ready for cutover. + +Run cross-project reconciliation in the operator's normal organisation context +so accessible organisational relation targets are included alongside private +projects. A narrower context can report a target as unavailable without proving +that it is absent. + +Database backups do not include separately stored file bytes. Pair the encrypted +database backup with `scripts/backup_task_project_files.py`: export file copies +against the restored snapshot, then restore those bytes into an isolated blob +directory beside the restored database. The manifest binds the database receipt, +projects, tasks, file identities, sizes and hashes. File restore refuses the +source database and verifies every restored file through canonical reads. Check +that representative restored tasks can still retrieve their original content +and attachments without Jira. + +Take the final paired snapshot after migration writes have settled. A database +dump spanning an in-progress file-copy creation can contain its concept before +its locator relations; a successful document restore alone does not establish +a usable content restore. + +`set_task_project_writer` records a reversible, per-project writer decision and +its evidence. Moving to `von` requires successful project reconciliation plus +site-retention, normal-use, restore and final-delta receipts. The smallest +delivery evidence includes ordinary browser/MCP operations for the affected +project, no unexplained missing issues or broken source relations, retrievable +original content, and a verified final delta. Missing source bytes, incorrect +visibility, lost native edits, duplicate identity, or an unverified recovery +path block the corresponding cutover claim. Merely wanting broader analytics, +typed sprint modelling or Jira UI parity does not. + +Until that decision is recorded, Jira remains the writer. Once it is `von`, +source importers refuse to overwrite or create task projections in that project; +coding agents use native task tracking and preserve Jira keys as aliases. Jira +access and retained archives remain available. Do not cancel Jira, delete its +data, or silently change collaborators' workflows as part of the writer switch. + +## Historical implementation slices + +The following records describe earlier bounded deliveries. Their completion +does not certify the current all-project migration. ## Scope diff --git a/scripts/backup_task_project_files.py b/scripts/backup_task_project_files.py new file mode 100644 index 000000000..6e50afa84 --- /dev/null +++ b/scripts/backup_task_project_files.py @@ -0,0 +1,226 @@ +"""Back up retained task/project files, or restore them beside a restored database. + +Complements restore_von_db.py: MongoDB contains file-copy identities, while the +original bytes live in blob storage. No Jira connection is needed for either +operation. Run export against the restored snapshot to bind its exact file set. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def file_copy_ids(value): + ids = set() + if isinstance(value, dict): + for key, item in value.items(): + if key == "file_copy_concept_id" and isinstance(item, str): + ids.add(item) + else: + ids.update(file_copy_ids(item)) + elif isinstance(value, list): + for item in value: + ids.update(file_copy_ids(item)) + return ids + + +def export_files(*, project_ids, destination, actor_id, database_receipt, cipher): + from src.backend.db.mongo_client import get_configured_database_name + from src.backend.db.repositories.concepts_repository import ConceptsRepository + from src.backend.services.computer_file_copy_service import ( + fetch_file_copy_bytes, + resolve_file_copy_blob_info, + ) + from src.backend.services.task_project_service import ( + get_task_project, + list_project_collections, + ) + + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + identifiers = set() + for project_id in project_ids: + project = get_task_project(project_id) + identifiers.update(file_copy_ids(project)) + identifiers.update(file_copy_ids(list_project_collections(project))) + tasks = list( + ConceptsRepository.find( + {"metadata.project_concept_id": {"$in": project_ids}}, + projection={"concept_id": 1, "metadata": 1}, + limit=0, + ) + ) + identifiers.update(file_copy_ids(tasks)) + manifest = { + "schema": "task_project_file_backup.v1", + "source_database": database_receipt["db_name"], + "export_database": get_configured_database_name(), + "database_receipt": database_receipt, + "project_concept_ids": project_ids, + "task_count": len(tasks), + "files": [], + "complete": False, + } + for identifier in sorted(identifiers): + result = fetch_file_copy_bytes( + file_copy_concept_id=identifier, user_concept_id=actor_id, allow_large=True + ) + if not result.get("success"): + raise ValueError(f"Cannot back up referenced file copy: {identifier}") + data = result["data"] + digest = hashlib.sha256(data).hexdigest() + encrypted_path = destination / f"{digest}.bin.enc" + encrypted_path.write_bytes(cipher.encrypt(data)) + os.chmod(encrypted_path, 0o600) + if ( + hashlib.sha256(cipher.decrypt(encrypted_path.read_bytes())).hexdigest() + != digest + ): + raise ValueError("Encrypted backup read-back failed") + info = resolve_file_copy_blob_info(file_copy_concept_id=identifier) + manifest["files"].append( + { + "file_copy_concept_id": identifier, + "sha256": digest, + "size_bytes": len(data), + "original_filename": info.original_filename, + "content_type": info.content_type, + } + ) + print( + json.dumps({"backed_up": identifier, "size_bytes": len(data)}), flush=True + ) + manifest["complete"] = True + (destination / "manifest.json").write_text(json.dumps(manifest, indent=2)) + return manifest + + +def restore_files(*, backup_dir, destination, actor_id, cipher): + from src.backend.db.mongo_client import get_configured_database_name + from src.backend.services.blob_store import LocalBlobStore + from src.backend.services.computer_file_copy_service import fetch_file_copy_bytes + from src.backend.services.concept_service import update_concept + + manifest = json.loads((backup_dir / "manifest.json").read_text()) + target_db = get_configured_database_name() + if target_db in {"von_db", manifest["source_database"]}: + raise ValueError("File restore requires a separate restored database") + if not manifest.get("complete"): + raise ValueError("File backup is incomplete") + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + store = LocalBlobStore(destination) + os.environ["VON_BLOB_STORE_LOCAL_ROOT"] = str(destination.resolve()) + os.environ["VON_BLOB_STORE_BACKEND"] = "local" + restored = [] + for entry in manifest["files"]: + digest = entry["sha256"] + if len(digest) != 64 or any(ch not in "0123456789abcdef" for ch in digest): + raise ValueError("Invalid file digest") + data = cipher.decrypt((backup_dir / f"{digest}.bin.enc").read_bytes()) + if ( + len(data) != entry["size_bytes"] + or hashlib.sha256(data).hexdigest() != digest + ): + raise ValueError("File backup integrity mismatch") + ref = store.put_bytes( + f"restored-task-files/{digest}", data, content_type=entry["content_type"] + ) + identifier = entry["file_copy_concept_id"] + update_concept( + identifier, + { + "attributes.file_copy_metadata_storage": "attributes.v1", + "attributes.blob_backend": "local", + "attributes.blob_key": ref.key, + "attributes.blob_uri": ref.uri, + "attributes.size_bytes": len(data), + "attributes.original_filename": entry["original_filename"], + "attributes.content_type": entry["content_type"], + }, + ) + result = fetch_file_copy_bytes( + file_copy_concept_id=identifier, user_concept_id=actor_id, allow_large=True + ) + if ( + not result.get("success") + or hashlib.sha256(result["data"]).hexdigest() != digest + ): + raise ValueError(f"Restored file canonical read-back failed: {identifier}") + restored.append(identifier) + receipt = { + "schema": "task_project_file_restore.v1", + "target_database": target_db, + "project_concept_ids": manifest["project_concept_ids"], + "task_count": manifest["task_count"], + "restored_file_copy_ids": restored, + "complete": True, + } + (destination / "restore-receipt.json").write_text(json.dumps(receipt, indent=2)) + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=["export", "restore"]) + parser.add_argument("--actor-concept-id", required=True) + parser.add_argument("--organisation-concept-id") + parser.add_argument("--project-concept-id", action="append", default=[]) + parser.add_argument("--database-receipt", type=Path) + parser.add_argument("--backup-dir", required=True, type=Path) + parser.add_argument("--restore-dir", type=Path) + args = parser.parse_args() + from cryptography.fernet import Fernet + + from src.backend.security.access_control import ( + force_access_control_enforcement, + override_current_actor, + ) + + cipher = Fernet(os.environ["VON_BACKUP_ENCRYPTION_KEY"].encode()) + with ( + override_current_actor(args.actor_concept_id, args.organisation_concept_id), + force_access_control_enforcement(), + ): + if args.operation == "export": + if not args.project_concept_id or not args.database_receipt: + parser.error( + "Export requires project IDs and the database backup receipt" + ) + result = export_files( + project_ids=args.project_concept_id, + destination=args.backup_dir, + actor_id=args.actor_concept_id, + cipher=cipher, + database_receipt=json.loads(args.database_receipt.read_text()), + ) + else: + if not args.restore_dir: + parser.error("Restore requires a separate destination directory") + result = restore_files( + backup_dir=args.backup_dir, + destination=args.restore_dir, + actor_id=args.actor_concept_id, + cipher=cipher, + ) + print( + json.dumps( + { + "complete": result["complete"], + "file_count": len( + result.get("files", result.get("restored_file_copy_ids", [])) + ), + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/backup_von_db.py b/scripts/backup_von_db.py index af032a81b..d26389210 100644 --- a/scripts/backup_von_db.py +++ b/scripts/backup_von_db.py @@ -522,7 +522,8 @@ def main(argv: list[str] | None = None) -> int: _ensure_dir(backup_root) # Write a prelude file at the backup root (helpful provenance when copying around). - (backup_root / "prelude.json").write_text( + # prelude.json is reserved by MongoDB Database Tools for server metadata. + (backup_root / "von_backup_prelude.json").write_text( json.dumps(asdict(prelude), indent=2, ensure_ascii=False) + "\n", encoding="utf-8", ) diff --git a/scripts/capture_jira_project_sources.py b/scripts/capture_jira_project_sources.py new file mode 100644 index 000000000..363a6310d --- /dev/null +++ b/scripts/capture_jira_project_sources.py @@ -0,0 +1,209 @@ +"""Capture source project/configuration evidence using the canonical Jira proxy. + +This command preserves source information and associations; it never switches +the writer or activates source workflows, permissions or automation rules. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + +async def capture(args): + from src.backend.integrations.internal_mcp.jira_proxy_mcp import get_jira_proxy + from src.backend.services.concept_service import update_concept + from src.backend.services.jira_source_retention_service import ( + capture_project, + capture_site_configuration, + store_source_archive, + ) + from src.backend.services.task_project_service import ( + associate_jira_source_collection, + ensure_jira_project, + ) + + proxy = await get_jira_proxy() + destination = args.output_dir + destination.mkdir(parents=True, exist_ok=True) + # The configured proxy account determines the source site, never a source + # document or a caller-provided credential-bearing destination. + source_site = proxy._auth_identity[0] + projects, errors = {}, [] + for key in dict.fromkeys(args.project_key): + if args.reuse_projects: + contexts = json.loads((destination / "project-contexts.json").read_text()) + projects[key] = contexts[key] + continue + archive = await capture_project(proxy, key) + (destination / f"project-{key}.json").write_text(json.dumps(archive, indent=2)) + if not archive.get("source", {}).get("id"): + errors.append({"project": key, "error": "project_source_unavailable"}) + continue + reference = store_source_archive( + archive, actor_concept_id=args.actor_concept_id + ) + org = ( + args.organisation_concept_id + if key == args.organisation_project_key + else None + ) + context = ensure_jira_project( + archive["source"], + actor_concept_id=args.actor_concept_id, + organisation_concept_id=org, + archive_reference=reference, + ) + projects[key] = context + (destination / "project-contexts.json").write_text( + json.dumps(projects, indent=2) + ) + if not archive["complete"]: + errors.append({"project": key, "error": "incomplete_project_source"}) + print( + json.dumps( + { + "project": key, + "complete": archive["complete"], + "project_concept_id": context["project_concept_id"], + } + ), + flush=True, + ) + if args.projects_only: + return {"projects": projects, "errors": errors} + site = await capture_site_configuration(proxy, site_url=source_site) + (destination / "site-source.json").write_text(json.dumps(site, indent=2)) + site_reference = store_source_archive(site, actor_concept_id=args.actor_concept_id) + (destination / "site-archive-reference.json").write_text( + json.dumps(site_reference, indent=2) + ) + for context in projects.values(): + update_concept( + context["project_concept_id"], + {"attributes.task_project.site_source_archive": site_reference}, + ) + collections = [] + resources = site["resources"] + for board in resources["boards"]["items"]: + board_id = str(board["id"]) + membership = resources.get(f"board_issues:{board_id}", {}) + covered = resources.get(f"board_projects:{board_id}", {}).get("items", []) + keys = {project["key"] for project in covered if project.get("key") in projects} + keys.update( + issue.get("fields", {}).get("project", {}).get("key") + for issue in membership.get("items", []) + ) + project_ids = [ + projects[key]["project_concept_id"] + for key in sorted(keys - {None}) + if key in projects + ] + if not project_ids: + continue + if not membership.get("complete"): + errors.append( + {"board_id": board_id, "error": "incomplete_board_membership"} + ) + continue + collection = associate_jira_source_collection( + name=board["name"], + source_id=board_id, + source_kind="board", + project_concept_ids=project_ids, + issue_keys=[issue["key"] for issue in membership["items"]], + archive_reference=site_reference, + actor_concept_id=args.actor_concept_id, + ) + collections.append(collection) + for source_filter in resources["filters"]["items"]: + filter_id = str(source_filter["id"]) + membership = resources.get(f"filter_issues:{filter_id}", {}) + keys = { + issue.get("fields", {}).get("project", {}).get("key") + for issue in membership.get("items", []) + } + project_ids = [ + projects[key]["project_concept_id"] + for key in sorted(keys - {None}) + if key in projects + ] + if not project_ids: + continue + if not membership.get("complete"): + errors.append( + {"filter_id": filter_id, "error": "incomplete_filter_membership"} + ) + continue + collections.append( + associate_jira_source_collection( + name=source_filter["name"], + source_id=filter_id, + source_kind="filter", + project_concept_ids=project_ids, + issue_keys=[issue["key"] for issue in membership["items"]], + archive_reference=site_reference, + actor_concept_id=args.actor_concept_id, + ) + ) + result = { + "projects": projects, + "collections": collections, + "site_archive": site_reference, + "errors": errors, + "cutover_ready": False, + } + (destination / "source-capture-report.json").write_text( + json.dumps(result, indent=2) + ) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--actor-concept-id", required=True) + parser.add_argument("--project-key", action="append", required=True) + parser.add_argument("--organisation-project-key", default="JVNAUTOSCI") + parser.add_argument("--organisation-concept-id") + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--projects-only", action="store_true") + parser.add_argument( + "--reuse-projects", + action="store_true", + help="Refresh site and collection evidence using previously captured project contexts.", + ) + args = parser.parse_args() + from src.backend.integrations.internal_mcp.gateway import ( + INTERNAL_MCP_TRUSTED_LOCAL_OPERATOR_SOURCE, + bind_internal_mcp_actor_context_source, + ) + from src.backend.security.access_control import ( + force_access_control_enforcement, + override_current_actor, + ) + + with ( + bind_internal_mcp_actor_context_source( + INTERNAL_MCP_TRUSTED_LOCAL_OPERATOR_SOURCE + ), + override_current_actor(args.actor_concept_id, None), + force_access_control_enforcement(), + ): + result = asyncio.run(capture(args)) + print( + json.dumps( + {"project_count": len(result["projects"]), "errors": result["errors"]} + ), + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/restore_von_db.py b/scripts/restore_von_db.py index 55ef57447..16b86f389 100644 --- a/scripts/restore_von_db.py +++ b/scripts/restore_von_db.py @@ -279,6 +279,27 @@ def _run_mongorestore( with tempfile.TemporaryDirectory(prefix="von-mongorestore-") as config_dir: config_root = Path(config_dir) os.chmod(config_root, 0o700) + restore_root = dump_root + legacy_prelude = dump_root / "prelude.json" + if legacy_prelude.is_file(): + prelude = json.loads(legacy_prelude.read_text(encoding="utf-8")) + if "mongo_uri_redacted" in prelude and "ServerVersion" not in prelude: + # Older Von archives used a filename now reserved by MongoDB. + # Stage only the selected database, preserving the source + # archive and avoiding a second byte copy on the same volume. + restore_root = config_root / "dump" + + def link_or_copy(source, destination): + try: + return os.link(source, destination) + except OSError: + return shutil.copy2(source, destination) + + shutil.copytree( + dump_root / source_db_name, + restore_root / source_db_name, + copy_function=link_or_copy, + ) config_path = config_root / "config.yml" config_path.write_text( f"uri: {json.dumps(mongo_uri)}\n", @@ -290,7 +311,7 @@ def _run_mongorestore( "--config", str(config_path), "--dir", - str(dump_root), + str(restore_root), "--nsFrom", f"{source_db_name}.*", "--nsTo", diff --git a/scripts/run_jira_task_migration.py b/scripts/run_jira_task_migration.py index c06182a33..005e85e63 100644 --- a/scripts/run_jira_task_migration.py +++ b/scripts/run_jira_task_migration.py @@ -51,6 +51,16 @@ def _parse_args() -> JiraTaskMigrationOptions: parser.add_argument("--source-migrated-label", default="migrated") parser.add_argument("--updated-within-hours", type=int) parser.add_argument("--include-done", action="store_true") + parser.add_argument( + "--preserve-source", + action="store_true", + help="Retain original issue/project content and complete activity/binaries in Von file copies.", + ) + parser.add_argument( + "--resume", + action="store_true", + help="Resume matching completed batches; changed source versions require a fresh delta run.", + ) parser.add_argument("--min-issue-number", type=int) parser.add_argument("--max-issue-number", type=int) parser.add_argument( @@ -73,7 +83,11 @@ def _parse_args() -> JiraTaskMigrationOptions: else None ), project_key=str(args.project_key).strip().upper(), - jql=(str(args.jql).strip() if isinstance(args.jql, str) and args.jql.strip() else None), + jql=( + str(args.jql).strip() + if isinstance(args.jql, str) and args.jql.strip() + else None + ), batch_size=int(args.batch_size), page_size=int(args.page_size), passes=int(args.passes), @@ -89,6 +103,8 @@ def _parse_args() -> JiraTaskMigrationOptions: else None ), include_done=bool(args.include_done), + preserve_source=bool(args.preserve_source), + resume=bool(args.resume), min_issue_number=( int(args.min_issue_number) if isinstance(args.min_issue_number, int) @@ -109,13 +125,32 @@ def main() -> None: ) options = _parse_args() + from src.backend.security.access_control import ( + override_current_actor, + force_access_control_enforcement, + ) + # This explicit local CLI is an operator entry point. Keep this authority # here, not in the runner also called by actor-bound durable workflows. - with bind_internal_mcp_actor_context_source( - INTERNAL_MCP_TRUSTED_LOCAL_OPERATOR_SOURCE + with ( + bind_internal_mcp_actor_context_source( + INTERNAL_MCP_TRUSTED_LOCAL_OPERATOR_SOURCE, + preexisting_actor_context=( + options.actor_concept_id, + options.organisation_concept_id, + ), + ), + override_current_actor( + options.actor_concept_id, options.organisation_concept_id + ), + force_access_control_enforcement(), ): report = run_jira_task_migration_sync(options) - print(json.dumps({"report_path": str(options.report_path), "summary": report}, indent=2)) + print( + json.dumps( + {"report_path": str(options.report_path), "summary": report}, indent=2 + ) + ) if __name__ == "__main__": diff --git a/src/backend/integrations/internal_mcp/catalogue.py b/src/backend/integrations/internal_mcp/catalogue.py index 7d082c0c0..a4b01f64c 100644 --- a/src/backend/integrations/internal_mcp/catalogue.py +++ b/src/backend/integrations/internal_mcp/catalogue.py @@ -35105,11 +35105,6 @@ def _resolve_task_actor_scope( "authenticated_actor_context_required", "Task effects require a trusted authenticated user.", ) - if not actor_scope.organisation_concept_id: - return None, make_error_response( - "authenticated_organisation_context_required", - "Task effects require a trusted organisation context.", - ) return actor_scope, None @@ -35118,14 +35113,14 @@ def _authorise_task_actor_mutation( *, surface: str, ) -> tuple[Any | None, dict[str, Any] | None, dict[str, Any] | None]: - """Require a same-organisation task owned by or assigned to the actor.""" + """Check task access, then the authority of the calling task surface.""" from ...services.task_management_service import TaskNotFoundError, get_task actor_scope, scope_error = _resolve_task_actor_scope(kwargs, surface=surface) if scope_error is not None: return None, None, scope_error - task_id = kwargs.get("task_concept_id") or kwargs.get("task_id") + task_id = _resolve_task_identifier(dict(kwargs)) if not isinstance(task_id, str) or not task_id.strip(): return ( actor_scope, @@ -35151,8 +35146,15 @@ def _authorise_task_actor_mutation( ) assert actor_scope is not None + from .gateway import internal_mcp_actor_context_is_trusted_local_operator + + # The configured local task principal can operate on accessible tasks, + # including personal projects and imported tasks with historical creators. + # get_task above enforces the principal's canonical visibility boundary. + if internal_mcp_actor_context_is_trusted_local_operator(): + return actor_scope, task, None actor_id = str(actor_scope.user_concept_id) - actor_org_id = str(actor_scope.organisation_concept_id) + actor_org_id = str(actor_scope.organisation_concept_id or "") task_org_id = str(task.get("organisation_concept_id") or "").strip() actor_controls_task = actor_id in { str(task.get("assignee_concept_id") or "").strip(), @@ -35166,7 +35168,7 @@ def _authorise_task_actor_mutation( "task_actor_scope_denied", ( "An ordinary-turn agent may change only a task in its authenticated " - "organisation that it created or is assigned to." + "organisation (or personal scope) that it created or is assigned to." ), ), ) @@ -35234,7 +35236,7 @@ def _task_create(**kwargs): idempotency_key = str(kwargs.get("idempotency_key") or "").strip() actor_id = str(actor_scope.user_concept_id) - actor_org_id = str(actor_scope.organisation_concept_id) + actor_org_id = str(actor_scope.organisation_concept_id or "") if ( str(assignee_id or "").strip() != actor_id or str(created_by or "").strip() != actor_id @@ -35426,6 +35428,8 @@ def _reconcile_after_create_failure() -> dict[str, Any] | None: evidence=evidence, notes=notes, reference_code=reference_code, + project_concept_id=kwargs.get("project_concept_id"), + collection_concept_ids=kwargs.get("collection_concept_ids"), agent_creation_fingerprint=creation_fingerprint, agent_creation_request_id=( request_id @@ -35491,7 +35495,7 @@ def _task_get(**kwargs): """Get a task by concept_id.""" from ...services.task_management_service import TaskNotFoundError, get_task - task_id = kwargs.get("task_concept_id") or kwargs.get("task_id") + task_id = _resolve_task_identifier(kwargs) if not task_id or not isinstance(task_id, str) or not task_id.strip(): return make_error_response( "MISSING_PARAM", @@ -35689,9 +35693,83 @@ def _resolve_task_identifier(payload: dict[str, Any]) -> str | None: if not isinstance(task_id, str): return None cleaned = task_id.strip() + if cleaned and not cleaned.startswith("#V#"): + from ...services.task_management_service import find_task_by_external_reference + + task = find_task_by_external_reference( + source_system="jira", external_id=cleaned.upper() + ) + if task: + return task["task_concept_id"] return cleaned or None +def _task_list_projects(**kwargs): + from ...services.task_project_service import list_task_projects + + return { + "success": True, + "projects": list_task_projects( + limit=kwargs.get("limit", 100), offset=kwargs.get("offset", 0) + ), + } + + +def _task_get_project(**kwargs): + from ...services.task_project_service import ( + get_task_project, + list_project_collections, + ) + + try: + project = get_task_project(kwargs.get("project_concept_id")) + return { + "success": True, + "project": project, + "collections": list_project_collections(project), + } + except ValueError as exc: + return make_error_response("NOT_FOUND", str(exc)) + + +def _task_get_collection(**kwargs): + from ...services.task_project_service import get_task_collection + + try: + return { + "success": True, + "collection": get_task_collection(kwargs.get("collection_concept_id")), + } + except ValueError as exc: + return make_error_response("NOT_FOUND", str(exc)) + + +def _task_get_source_archive(**kwargs): + from ...services.jira_source_retention_service import get_retained_task_source + + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="task_get_source_archive", + require_actor=True, + ) + if denial: + return denial + try: + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + return { + "success": True, + **get_retained_task_source( + _resolve_task_identifier(kwargs), + actor_concept_id=actor_scope.user_concept_id, + resource=kwargs.get("resource"), + offset=kwargs.get("offset", 0), + limit=kwargs.get("limit", 100), + ), + } + except ValueError as exc: + return make_error_response("NOT_FOUND", str(exc)) + + def _task_search(**kwargs): """Search tasks with Jira-like rich filtering.""" from ...services.task_management_service import ( @@ -35702,6 +35780,8 @@ def _task_search(**kwargs): try: result = search_tasks( query=kwargs.get("query"), + project_concept_id=kwargs.get("project_concept_id"), + collection_concept_id=kwargs.get("collection_concept_id"), status_filter=kwargs.get("status_filter") or kwargs.get("status"), statuses=kwargs.get("statuses"), task_type_ids=kwargs.get("task_type_ids") or kwargs.get("task_type_id"), @@ -36144,6 +36224,11 @@ def _task_import_jira_issues(**kwargs): participant_map: dict[str, str] = {} participant_map.update(assignee_map) participant_map.update(jira_map) + preserve_source = _coerce_bool_input(kwargs.get("preserve_source"), default=False) + if preserve_source and not actor_concept_id: + return make_error_response( + "MISSING_PARAM", "Source preservation requires a resolved actor." + ) async def _fetch_jira_issues() -> dict[str, Any]: proxy = await get_jira_proxy() @@ -36204,7 +36289,66 @@ async def _fetch_jira_issues() -> dict[str, Any]: issue_docs: list[dict[str, Any]] = [] attachment_max_size_bytes = _jira_attachment_max_size_bytes() + project_contexts: dict[str, Any] = {} for issue_key in discovered_keys: + if preserve_source: + from ...services.jira_source_retention_service import ( + capture_issue, + capture_project, + store_source_archive, + ) + from ...services.task_project_service import ensure_jira_project + + try: + archive, issue_doc = await capture_issue( + proxy, + issue_key, + attachment_max_size_bytes=attachment_max_size_bytes, + ) + project_key = issue_doc["fields"]["project"]["key"] + if not dry_run: + # Original source copies belong to the importing actor. + # Operational project membership does not publish raw data. + issue_doc["_von_source_archive"] = store_source_archive( + archive, actor_concept_id=actor_concept_id + ) + if project_key not in project_contexts: + project_archive = await capture_project(proxy, project_key) + reference = store_source_archive( + project_archive, actor_concept_id=actor_concept_id + ) + if not project_archive["complete"]: + fetch_errors.append( + { + "issue_key": issue_key, + "error": f"project_source_retention_incomplete:{project_key}", + } + ) + project_contexts[project_key] = ensure_jira_project( + project_archive["source"], + actor_concept_id=actor_concept_id, + archive_reference=reference, + organisation_concept_id=organisation_concept_id, + ) + issue_doc["_von_project_context"] = project_contexts[ + project_key + ] + if not archive["complete"]: + fetch_errors.append( + { + "issue_key": issue_key, + "error": "source_retention_incomplete", + } + ) + issue_docs.append(issue_doc) + except Exception as exc: + fetch_errors.append( + { + "issue_key": issue_key, + "error": f"source_capture_failed:{type(exc).__name__}:{exc}", + } + ) + continue seeded_issue_doc = seeded_issue_docs.get(issue_key) issue_doc: dict[str, Any] | None = None if isinstance(seeded_issue_doc, Mapping): @@ -36597,6 +36741,25 @@ async def _sync_source_labels_to_jira() -> dict[str, Any]: } report["success"] = bool(report.get("success", True)) + if preserve_source and fetch_payload.get("fetch_errors"): + report["success"] = False + report["error_code"] = "source_retention_incomplete" + if preserve_source: + report["source_retention"] = { + "mode": "original_source_and_binary_archives", + "archives": [ + { + "issue_key": item.get("key"), + "reference": item.get("_von_source_archive"), + } + for item in issue_docs + ], + "cutover_ready": False, + "reason": "Batch capture does not establish estate reconciliation, usability or restoration.", + } + if report.get("summary", {}).get("errors", 0): + report["success"] = False + report.setdefault("error_code", "task_projection_incomplete") if int(source_label_sync_report.get("error_count", 0)) > 0: report["success"] = False report["source_label_sync"] = source_label_sync_report @@ -37250,6 +37413,40 @@ def _task_add_attachment(**kwargs): task_id = _resolve_task_identifier(kwargs) filename = kwargs.get("filename") or kwargs.get("name") uri = kwargs.get("uri") + if kwargs.get("content_base64") is not None: + import base64 + from ...services.task_management_service import add_task_attachment_bytes + + actor_scope, denial = _resolve_internal_mcp_scoped_assertion_actor_scope( + kwargs, + surface="task_add_attachment", + require_actor=True, + ) + if denial: + return denial + try: + encoded = kwargs["content_base64"] + if not isinstance(encoded, str) or len(encoded) > 140 * 1024 * 1024: + raise ValueError("Attachment exceeds the 100 MiB limit") + with _bind_internal_mcp_scoped_assertion_actor(actor_scope): + attachment = add_task_attachment_bytes( + task_id, + data=base64.b64decode(encoded, validate=True), + filename=filename, + actor_concept_id=actor_scope.user_concept_id, + media_type=kwargs.get("media_type"), + note=kwargs.get("note"), + ) + return { + "success": True, + "task_concept_id": task_id, + "attachment": attachment, + } + except (ValueError, InvalidTaskDataError) as exc: + return make_error_response("INVALID_DATA", str(exc)) + except TaskNotFoundError as exc: + return make_error_response("NOT_FOUND", str(exc)) + if not task_id: return make_error_response( "MISSING_PARAM", @@ -44520,6 +44717,63 @@ def _build_default_catalogue_task_and_workflow_definitions() -> List[MethodDefin "subjects and bodies, with sent/received and paging filters." ), ), + MethodDefinition( + name="task_list_projects", + handler=_task_list_projects, + input_schema=Schema( + required={}, + optional={"limit": (int,), "offset": (int,)}, + allow_unknown=False, + ), + output_schema=_task_generic_output_schema("projects"), + category="read", + description="List accessible task projects and their collection references, source identities and current writer.", + ), + MethodDefinition( + name="task_get_source_archive", + handler=_task_get_source_archive, + input_schema=Schema( + required={"task_concept_id": (str,)}, + optional={ + "resource": (str,), + "offset": (int,), + "limit": (int,), + "acting_user_concept_id": (str,), + "namespace": (str,), + }, + allow_unknown=False, + ), + output_schema=_task_generic_output_schema("source_archive"), + category="read", + ordinary_turn_trusted_argument_bindings={ + "acting_user_concept_id": "actor_user_concept_id" + }, + description="Read a checksum-verified retained Jira original or page of original comments, worklogs or changelog, without contacting Jira. Includes original-file download reference; private archive access remains actor-scoped.", + ), + MethodDefinition( + name="task_get_project", + handler=_task_get_project, + input_schema=Schema( + required={"project_concept_id": (str,)}, + optional={}, + allow_unknown=False, + ), + output_schema=_task_generic_output_schema("project"), + category="read", + description="Get a task project and its accessible task collections.", + ), + MethodDefinition( + name="task_get_collection", + handler=_task_get_collection, + input_schema=Schema( + required={"collection_concept_id": (str,)}, + optional={}, + allow_unknown=False, + ), + output_schema=_task_generic_output_schema("collection"), + category="read", + description="Get a task collection, its selection semantics and associated projects.", + ), MethodDefinition( name="task_create", handler=_task_create, @@ -44529,6 +44783,8 @@ def _build_default_catalogue_task_and_workflow_definitions() -> List[MethodDefin "description": str, }, optional={ + "project_concept_id": (str, type(None)), + "collection_concept_ids": (list, type(None)), "assignee_id": (str, type(None)), "assignee_concept_id": (str, type(None)), "originating_session_id": (str, type(None)), @@ -44658,6 +44914,8 @@ def _build_default_catalogue_task_and_workflow_definitions() -> List[MethodDefin input_schema=Schema( required={}, optional={ + "project_concept_id": (str, type(None)), + "collection_concept_id": (str, type(None)), "query": (str, type(None)), "status_filter": (str, type(None)), "status": (str, type(None)), @@ -44721,6 +44979,7 @@ def _build_default_catalogue_task_and_workflow_definitions() -> List[MethodDefin input_schema=Schema( required={}, optional={ + "preserve_source": (bool,), "issue_keys": (list, type(None)), "jql": (str, type(None)), "max_results": (int, type(None)), @@ -45136,6 +45395,8 @@ def _build_default_catalogue_task_and_workflow_definitions() -> List[MethodDefin "filename": (str,), "name": (str,), "uri": (str,), + "content_base64": (str,), + "acting_user_concept_id": (str,), "media_type": (str, type(None)), "mime_type": (str, type(None)), "size_bytes": (int, type(None)), @@ -45145,7 +45406,7 @@ def _build_default_catalogue_task_and_workflow_definitions() -> List[MethodDefin "namespace": (str, type(None)), }, allow_unknown=True, - description="Add attachment metadata to a task.", + description="Upload a task attachment using content_base64, or add an existing URI.", ), output_schema=task_add_attachment_output_schema, category="write", diff --git a/src/backend/integrations/internal_mcp/jira_migration_resources.py b/src/backend/integrations/internal_mcp/jira_migration_resources.py new file mode 100644 index 000000000..9e93748b0 --- /dev/null +++ b/src/backend/integrations/internal_mcp/jira_migration_resources.py @@ -0,0 +1,133 @@ +"""Named, read-only Jira source resources needed by the existing importer. + +Paths are relative to the configured Jira site. Source-provided URLs are never +used as credential-bearing request destinations. Source configuration is data; +reading a scheme, role or automation reference does not activate it in Von. +""" + +from urllib.parse import quote +from uuid import UUID + +# API family, path, pagination. Keep the source names in retention manifests. +RESOURCES = { + "projects": ("api/3", "project/search", True), + "project": ("api/3", "project/{id}", False), + "components": ("api/3", "project/{id}/components", False), + "versions": ("api/3", "project/{id}/versions", False), + "project_properties": ("api/3", "project/{id}/properties", False), + "project_property": ("api/3", "project/{id}/properties/{secondary}", False), + "project_roles": ("api/3", "project/{id}/role", False), + "project_role": ("api/3", "project/{id}/role/{secondary}", False), + "project_statuses": ("api/3", "project/{id}/statuses", False), + "permission_scheme": ("api/3", "project/{id}/permissionscheme", False), + "security_scheme": ("api/3", "project/{id}/issuesecuritylevelscheme", False), + "notification_scheme": ("api/3", "project/{id}/notificationscheme", False), + "workflow_scheme": ("api/3", "workflowscheme/project", False), + "issue_type_scheme": ("api/3", "issuetypescheme/project", True), + "field_configuration_scheme": ("api/3", "fieldconfigurationscheme/project", True), + "screen_scheme": ("api/3", "issuetypescreenscheme/project", True), + "fields": ("api/3", "field", False), + "field_contexts": ("api/3", "field/{id}/context", True), + "field_options": ("api/3", "field/{id}/context/{secondary}/option", True), + "field_defaults": ("api/3", "field/{id}/context/defaultValue", True), + "issue_types": ("api/3", "issuetype", False), + "statuses": ("api/3", "status", False), + "resolutions": ("api/3", "resolution", False), + "priorities": ("api/3", "priority", False), + "workflows": ("api/3", "workflow/search", True), + "screens": ("api/3", "screens", True), + "filters": ("api/3", "filter/search", True), + "filter_issues": ("api/3", "search/jql", True), + "filter": ("api/3", "filter/{id}", False), + "dashboards": ("api/3", "dashboard/search", True), + "dashboard": ("api/3", "dashboard/{id}", False), + "dashboard_gadgets": ("api/3", "dashboard/{id}/gadget", False), + "webhooks": ("api/3", "webhook", True), + "boards": ("agile/1.0", "board", True), + "board_issues": ("software/1.0", "board/{id}/issue", True), + "board_properties": ("agile/1.0", "board/{id}/properties", False), + "board_property": ("agile/1.0", "board/{id}/properties/{secondary}", False), + "board_quick_filters": ("agile/1.0", "board/{id}/quickfilter", True), + "board_configuration": ("agile/1.0", "board/{id}/configuration", False), + "board_sprints": ("agile/1.0", "board/{id}/sprint", True), + "board_projects": ("agile/1.0", "board/{id}/project", True), + "comments": ("api/3", "issue/{id}/comment", True), + "worklogs": ("api/3", "issue/{id}/worklog", True), + "changelog": ("api/3", "issue/{id}/changelog", True), + "remote_links": ("api/3", "issue/{id}/remotelink", False), + "issue_properties": ("api/3", "issue/{id}/properties", False), + "issue_property": ("api/3", "issue/{id}/properties/{secondary}", False), + "votes": ("api/3", "issue/{id}/votes", False), +} + + +def resource_request( + resource, + identifier="", + secondary_id="", + start_at=0, + max_results=100, + next_page_token=None, +): + try: + family, template, paginated = RESOURCES[resource] + except KeyError as exc: + raise ValueError("Unsupported Jira migration resource") from exc + for marker, value in (("{id}", identifier), ("{secondary}", secondary_id)): + if marker in template and ( + not isinstance(value, str) or not value or value in {".", ".."} + ): + raise ValueError("Missing or invalid resource identifier") + path = template.format( + id=quote(str(identifier), safe=""), secondary=quote(str(secondary_id), safe="") + ) + params = {} + if paginated: + params.update( + startAt=max(0, int(start_at)), maxResults=max(1, min(int(max_results), 100)) + ) + if resource in {"board_issues", "filter_issues"}: + params.pop("startAt", None) + params["fields"] = "key,project,updated" + if next_page_token: + params["nextPageToken"] = next_page_token + if resource == "filter_issues": + if not str(identifier).isdigit(): + raise ValueError("A Jira filter ID is required") + params["jql"] = f"filter = {identifier} ORDER BY key ASC" + if resource == "project": + params["expand"] = ( + "description,lead,issueTypes,url,projectKeys,permissions,insight" + ) + elif resource == "permission_scheme": + params["expand"] = "permissions,user,group,projectRole,field,all" + elif resource == "notification_scheme": + params["expand"] = "all" + elif resource in { + "workflow_scheme", + "issue_type_scheme", + "field_configuration_scheme", + "screen_scheme", + }: + if not str(identifier).isdigit(): + raise ValueError("A Jira project ID is required") + params["projectId"] = identifier + elif resource == "workflows": + params["expand"] = "transitions,statuses,operations,schemes" + elif resource == "filters": + params["expand"] = "jql,description,owner,sharePermissions,editPermissions" + return f"/rest/{family}/{path}", params + + +def export_download_path(kind, export_id, cloud_id=None): + """Resolve only already-created exports on the configured source site.""" + export_id = str(UUID(str(export_id))) + if kind == "cloud_backup": + return f"/plugins/servlet/export/download/?fileId={export_id}" + if kind == "automation": + cloud_id = str(UUID(str(cloud_id))) + return ( + f"/gateway/api/automation/internal-api/jira/{cloud_id}/pro/rest/" + f"GLOBAL/rule/export/{export_id}/download" + ) + raise ValueError("Unsupported Jira export kind") diff --git a/src/backend/integrations/internal_mcp/jira_proxy_mcp.py b/src/backend/integrations/internal_mcp/jira_proxy_mcp.py index 6b26d5644..fe4e1c4fa 100644 --- a/src/backend/integrations/internal_mcp/jira_proxy_mcp.py +++ b/src/backend/integrations/internal_mcp/jira_proxy_mcp.py @@ -203,6 +203,18 @@ def __init__(self, config: JiraProxyConfig): log_tag=_LOG_TAG, ) self._client = MCPStdIOClient(client_config) + # Export downloads use the existing three 60-second HTTP attempts. + # Allow twice that declared inner bound for transfer and JSON encoding; + # ordinary issue reads retain their current operation budget. + self._export_client = MCPStdIOClient( + MCPServerConfig( + command=config.command, + args=config.args, + env=config.env, + timeout_sec=360.0, + log_tag=_LOG_TAG, + ) + ) async def _call( self, @@ -219,7 +231,12 @@ async def _call( resource_id=resource_id, ) try: - result = await self._client.call_tool(tool_name, arguments) + client = ( + self._export_client + if tool_name == "jira_get_migration_export" + else self._client + ) + result = await client.call_tool(tool_name, arguments) if authority is not None and isinstance(result, dict): result = {**result, "authority": authority} return result @@ -286,6 +303,41 @@ async def get_comments( async def get_watchers(self, *, issue_key: str) -> Dict[str, Any]: return await self._call("jira_get_watchers", {"issue_key": issue_key}) + async def get_migration_resource( + self, + *, + resource: str, + identifier: str = "", + secondary_id: str = "", + start_at: int = 0, + max_results: int = 100, + next_page_token: str | None = None, + ) -> Any: + """Read a named source resource through the existing account boundary.""" + return await self._call( + "jira_get_migration_resource", + { + "resource": resource, + "identifier": identifier, + "secondary_id": secondary_id, + "start_at": start_at, + "max_results": max_results, + "next_page_token": next_page_token, + }, + ) + + async def get_migration_export( + self, *, kind: str, export_id: str, cloud_id: str | None = None + ): + return await self._call( + "jira_get_migration_export", + { + "kind": kind, + "export_id": export_id, + "cloud_id": cloud_id, + }, + ) + async def get_attachment_content( self, *, diff --git a/src/backend/integrations/internal_mcp/tool_contract_registry.py b/src/backend/integrations/internal_mcp/tool_contract_registry.py index 43f322be0..0c102c89d 100644 --- a/src/backend/integrations/internal_mcp/tool_contract_registry.py +++ b/src/backend/integrations/internal_mcp/tool_contract_registry.py @@ -482,6 +482,15 @@ def _contract_from_method_definition( "input_schema": { "type": "object", "properties": { + "project_concept_id": { + "type": "string", + "description": "Project selected from task_list_projects.", + }, + "collection_concept_ids": { + "type": "array", + "items": {"type": "string"}, + "description": "Collections associated with the selected project.", + }, "title": { "type": "string", "description": "Brief task title (required)", diff --git a/src/backend/mcp_server/mcp_server.py b/src/backend/mcp_server/mcp_server.py index be972f549..e5668c63c 100644 --- a/src/backend/mcp_server/mcp_server.py +++ b/src/backend/mcp_server/mcp_server.py @@ -3,6 +3,8 @@ import json import re import time +import sys +from pathlib import Path from typing import Any, Dict, List, Sequence import anyio @@ -13,6 +15,10 @@ from mcp.server.stdio import stdio_server import mcp.types as types +# The Jira proxy launches this file directly, rather than with python -m. +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + # --------------------------------------------------------- # Load env # --------------------------------------------------------- @@ -992,6 +998,22 @@ def _flatten_comment_page(page: Dict[str, Any]) -> Dict[str, Any]: @server.list_tools() async def list_tools() -> List[types.Tool]: return [ + types.Tool( + name="jira_get_migration_resource", + description="Read a named Jira migration source resource; no arbitrary URLs or writes.", + inputSchema={ + "type": "object", + "properties": { + "resource": {"type": "string"}, + "identifier": {"type": "string"}, + "secondary_id": {"type": "string"}, + "start_at": {"type": "integer"}, + "max_results": {"type": "integer"}, + "next_page_token": {"type": ["string", "null"]}, + }, + "required": ["resource"], + }, + ), types.Tool( name="jira_search", description="Run a JQL query in Jira and return matching issues.", @@ -1116,6 +1138,20 @@ async def list_tools() -> List[types.Tool]: "required": ["attachment_id"], }, ), + types.Tool( + name="jira_get_migration_export", + description="Download an already-created Jira cloud backup or Automation export from the configured site.", + inputSchema={ + "type": "object", + "properties": { + "kind": {"type": "string", "enum": ["cloud_backup", "automation"]}, + "export_id": {"type": "string"}, + "cloud_id": {"type": ["string", "null"]}, + }, + "required": ["kind", "export_id"], + "additionalProperties": False, + }, + ), types.Tool( name="jira_get_transitions", description=( @@ -1359,6 +1395,15 @@ async def call_tool( text = _tool_json(result) return [types.TextContent(type="text", text=text)] + elif name == "jira_get_migration_resource": + from src.backend.integrations.internal_mcp.jira_migration_resources import ( + resource_request, + ) + + path, params = resource_request(**arguments) + result = _request_json("GET", f"{JIRA_BASE_URL}{path}", params=params) + return [types.TextContent(type="text", text=_tool_json(result))] + elif name == "jira_get_issue": issue_key = arguments["issue_key"] issue_params: Dict[str, Any] | None = None @@ -1399,6 +1444,26 @@ async def call_tool( text = _tool_json(result) return [types.TextContent(type="text", text=text)] + elif name == "jira_get_migration_export": + from src.backend.integrations.internal_mcp.jira_migration_resources import ( + export_download_path, + ) + + path = export_download_path( + arguments["kind"], arguments["export_id"], arguments.get("cloud_id") + ) + result = _request_bytes(JIRA_BASE_URL + path) + content = result.pop("content", None) + if isinstance(content, bytes): + if "text/html" in str(result.get("content_type", "")): + result.update( + success=False, error="export_returned_html_instead_of_source" + ) + else: + result["content_base64"] = base64.b64encode(content).decode("ascii") + result["size_bytes"] = len(content) + return [types.TextContent(type="text", text=_tool_json(result))] + elif name == "jira_get_attachment_content": attachment_id = arguments["attachment_id"] raw_max_size = arguments.get("max_size_bytes") diff --git a/src/backend/mcp_server/mcp_stdio_server.py b/src/backend/mcp_server/mcp_stdio_server.py index 13ce7be9c..0d9ad0214 100644 --- a/src/backend/mcp_server/mcp_stdio_server.py +++ b/src/backend/mcp_server/mcp_stdio_server.py @@ -1028,7 +1028,9 @@ async def list_tools() -> list[Tool]: } ) -# Only these read handlers inherit the local stdio operator boundary. Keep +# Only these named handlers inherit the local stdio operator boundary. Jira +# writes still pass their existing write-profile/effect checks; this binding +# permits their nested actor-scoped read preflight. Keep # execution, recovery, schedules and ontology mutations on their own authority # paths; an input flag or actor identifier must never enlarge this set. _TRUSTED_LOCAL_OPERATOR_DIAGNOSTIC_READ_TOOLS = frozenset( @@ -1038,8 +1040,15 @@ async def list_tools() -> list[Tool]: "conversation_inspect_batch", "conversation_telemetry_get_locator", "conversation_transcript_page", + "task_get_source_archive", + "task_add_attachment", "jira_get_comments", "jira_get_issue", + "jira_get_myself", + "jira_get_transitions", + "jira_get_project_issue_types", + "jira_create_issue", + "jira_update_issue", "jira_search", "turn_execution_get", "turn_execution_get_diagnostics", @@ -1202,6 +1211,80 @@ async def call_tool(name: str, arguments: Any) -> list[TextContent]: # type: ig ] try: + if name in _NATIVE_TASK_TOOLS or name in { + "create_task", + "get_task", + "list_my_tasks", + "assign_task", + "update_task_status", + }: + configured_actor = os.environ.get( + "VON_MCP_TASK_ACTOR_CONCEPT_ID", "" + ).strip() + configured_org = ( + os.environ.get("VON_MCP_TASK_ORGANISATION_CONCEPT_ID", "").strip() + or None + ) + inherited_source = ( + internal_mcp_gateway_module.get_internal_mcp_actor_context_source() + ) + if configured_actor and inherited_source is None: + # A local server installation may bind its task tools to its + # operator once. Per-call payloads cannot select a principal; + # this also lets native task work continue with Jira offline. + if not configured_actor.startswith("#V#"): + return [ + _json_error( + "Invalid configured task actor", + error_code="invalid_task_actor_configuration", + ) + ] + if configured_org and not configured_org.startswith("#V#"): + return [ + _json_error( + "Invalid configured task organisation", + error_code="invalid_task_actor_configuration", + ) + ] + actor_fields = ( + "acting_user_concept_id", + "actor_concept_id", + "created_by_concept_id", + "creator_concept_id", + "added_by_concept_id", + "author_concept_id", + "user_concept_id", + ) + if any( + parsed_arguments.get(key) not in (None, "", configured_actor) + for key in actor_fields + ): + return [ + _json_error( + "Task actor differs from the configured local principal", + error_code="task_actor_mismatch", + ) + ] + from src.backend.security.access_control import ( + override_current_actor, + force_access_control_enforcement, + ) + + task_arguments = dict(parsed_arguments) + task_arguments.update({key: configured_actor for key in actor_fields}) + if not task_arguments.get("request_id"): + from uuid import uuid4 + + task_arguments["request_id"] = f"task-mcp-{uuid4().hex}" + with ( + override_current_actor(configured_actor, configured_org), + force_access_control_enforcement(), + internal_mcp_gateway_module.bind_internal_mcp_actor_context_source( + internal_mcp_gateway_module.INTERNAL_MCP_TRUSTED_LOCAL_OPERATOR_SOURCE, + preexisting_actor_context=(configured_actor, configured_org), + ), + ): + return await handler(task_arguments) governed_method = _STDIO_GOVERNED_ONTOLOGY_METHODS.get(name) if governed_method: # Stdio has no independently authenticated actor context. Binding @@ -4617,6 +4700,8 @@ async def _handle_create_task(arguments: dict[str, Any]) -> list[TextContent]: evidence=arguments.get("evidence"), notes=arguments.get("notes"), reference_code=arguments.get("reference_code"), + project_concept_id=arguments.get("project_concept_id"), + collection_concept_ids=arguments.get("collection_concept_ids"), ) return [_json_text({"success": True, **result})] except Exception as exc: @@ -4723,7 +4808,45 @@ async def _handle_otter_collection_status(arguments: dict[str, Any]) -> list[Tex return [_json_text(call_collection("status", **arguments))] +def _native_task_handler(name): + async def handle(arguments): + from src.backend.integrations.internal_mcp import catalogue + + # Reuse the canonical task handlers. The stdio write profile is checked + # by call_tool before reaching this adapter; actor-bound callers retain + # their context and ordinary continuation checks. + result = getattr(catalogue, f"_{name}")(**arguments) + return [_json_text(result)] + + return handle + + +_NATIVE_TASK_TOOLS = ( + "task_get", + "task_search", + "task_update_fields", + "task_get_transitions", + "task_transition", + "task_create_subtask", + "task_set_parent", + "task_link", + "task_unlink", + "task_add_comment", + "task_list_comments", + "task_add_attachment", + "task_list_attachments", + "task_add_worklog", + "task_list_worklog", + "task_get_history", + "task_list_projects", + "task_get_project", + "task_get_collection", + "task_get_source_archive", +) + + _TOOL_HANDLERS: dict[str, Callable[[dict[str, Any]], Awaitable[list[TextContent]]]] = { + **{name: _native_task_handler(name) for name in _NATIVE_TASK_TOOLS}, "otter_collect_now": _handle_otter_collect_now, "otter_collection_status": _handle_otter_collection_status, "get_context": _handle_get_context, diff --git a/src/backend/mcp_server/vontology_mcp.json b/src/backend/mcp_server/vontology_mcp.json index 64fbea9eb..92e6c0efb 100644 --- a/src/backend/mcp_server/vontology_mcp.json +++ b/src/backend/mcp_server/vontology_mcp.json @@ -1494,6 +1494,17 @@ "inputSchema": { "type": "object", "properties": { + "project_concept_id": { + "type": "string", + "description": "Project selected from task_list_projects." + }, + "collection_concept_ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Collections associated with the selected project." + }, "title": { "type": "string", "description": "Brief task title (required)" @@ -5861,6 +5872,1143 @@ "description": "skill_catalogue_sync input: optional include_default_roots (defaults true), optional per-scope root overrides, and optional dry_run to preview sync without mutating Vontology." } }, + { + "name": "task_add_attachment", + "description": "Record task attachment metadata (filename/URI/media type/size) in a Jira-like attachment operation.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "name": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "content_base64": { + "type": "string" + }, + "acting_user_concept_id": { + "type": "string" + }, + "media_type": { + "type": [ + "string", + "null" + ] + }, + "mime_type": { + "type": [ + "string", + "null" + ] + }, + "size_bytes": { + "type": [ + "integer", + "null" + ] + }, + "note": { + "type": [ + "string", + "null" + ] + }, + "added_by_concept_id": { + "type": [ + "string", + "null" + ] + }, + "author_concept_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "Upload a task attachment using content_base64, or add an existing URI." + } + }, + { + "name": "task_add_comment", + "description": "Add a retry-safe comment to a Von task created by or assigned to the authenticated actor in the same organisation.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "body": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "author_concept_id": { + "type": [ + "string", + "null" + ] + }, + "acting_user_concept_id": { + "type": [ + "string", + "null" + ] + }, + "organisation_concept_id": { + "type": [ + "string", + "null" + ] + }, + "request_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "task_concept_id", + "body" + ], + "additionalProperties": true, + "description": "Add an idempotent comment to an actor-owned task.", + "x-von-argument-aliases": { + "task_id": "task_concept_id", + "comment": "body" + } + } + }, + { + "name": "task_add_worklog", + "description": "Add a task worklog entry for effort tracking.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "time_spent_minutes": { + "type": "integer" + }, + "comment": { + "type": [ + "string", + "null" + ] + }, + "started_at": { + "type": [ + "string", + "null" + ] + }, + "author_concept_id": { + "type": [ + "string", + "null" + ] + }, + "added_by_concept_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "Add a worklog entry for a task." + } + }, + { + "name": "task_create_subtask", + "description": "Create a subtask under an existing parent task using the same core task creation pathway.", + "inputSchema": { + "type": "object", + "properties": { + "parent_task_concept_id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "assignee_concept_id": { + "type": [ + "string", + "null" + ] + }, + "assignee_id": { + "type": [ + "string", + "null" + ] + }, + "originating_session_id": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": [ + "string", + "null" + ] + }, + "created_by_concept_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "priority": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "critical" + ] + }, + "start_date": { + "type": [ + "string", + "null" + ] + }, + "due_date": { + "type": [ + "string", + "null" + ] + }, + "epic_task_concept_id": { + "type": [ + "string", + "null" + ] + }, + "organisation_concept_id": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "parent_task_concept_id", + "title", + "description" + ], + "additionalProperties": true, + "description": "Create a task and attach it as a subtask of a parent task.", + "x-von-scalar-source-fields": { + "parent_task_concept_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ] + } + } + }, + { + "name": "task_get", + "description": "Retrieve a Von task by its concept_id. Returns title, description, status, priority, assignee, and other metadata.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + } + }, + "required": [], + "additionalProperties": true, + "description": "Get a task by its concept_id.", + "x-von-scalar-source-fields": { + "task_concept_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ], + "task_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ] + } + } + }, + { + "name": "task_get_collection", + "description": "Get a task collection, its selection semantics and associated projects.", + "inputSchema": { + "type": "object", + "properties": { + "collection_concept_id": { + "type": "string" + } + }, + "required": [ + "collection_concept_id" + ], + "additionalProperties": false + } + }, + { + "name": "task_get_history", + "description": "Return task history/audit timeline.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "limit": { + "type": [ + "integer", + "null" + ] + }, + "offset": { + "type": [ + "integer", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "Get task audit history/timeline entries." + } + }, + { + "name": "task_get_project", + "description": "Get a task project and its accessible task collections.", + "inputSchema": { + "type": "object", + "properties": { + "project_concept_id": { + "type": "string" + } + }, + "required": [ + "project_concept_id" + ], + "additionalProperties": false + } + }, + { + "name": "task_get_source_archive", + "description": "Read a checksum-verified retained Jira original or page of original comments, worklogs or changelog, without contacting Jira. Includes original-file download reference; private archive access remains actor-scoped.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "resource": { + "type": "string" + }, + "offset": { + "type": "integer" + }, + "limit": { + "type": "integer" + }, + "acting_user_concept_id": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "task_concept_id" + ], + "additionalProperties": false + } + }, + { + "name": "task_get_transitions", + "description": "Return available task status transitions (Jira-style), including transition IDs and resulting statuses.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + } + }, + "required": [], + "additionalProperties": true, + "description": "List available transitions for the task's current status.", + "x-von-scalar-source-fields": { + "task_concept_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ], + "task_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ] + } + } + }, + { + "name": "task_link", + "description": "Create typed task dependencies (e.g., depends_on, blocks, relates_to) for Jira-like task graph management.", + "inputSchema": { + "type": "object", + "properties": { + "source_task_concept_id": { + "type": "string" + }, + "target_task_concept_id": { + "type": "string" + }, + "link_type": { + "type": "string" + }, + "source_task_id": { + "type": [ + "string", + "null" + ] + }, + "target_task_id": { + "type": [ + "string", + "null" + ] + }, + "actor_concept_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "source_task_concept_id", + "target_task_concept_id", + "link_type" + ], + "additionalProperties": true, + "description": "Create a typed dependency link between two tasks." + } + }, + { + "name": "task_list_attachments", + "description": "List task attachments with offset/limit pagination.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "limit": { + "type": [ + "integer", + "null" + ] + }, + "offset": { + "type": [ + "integer", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "List attachments recorded on a task." + } + }, + { + "name": "task_list_comments", + "description": "List task comments with offset/limit pagination.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "limit": { + "type": [ + "integer", + "null" + ] + }, + "offset": { + "type": [ + "integer", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "List comments for a task." + } + }, + { + "name": "task_list_projects", + "description": "List accessible task projects and their collection references, source identities and current writer.", + "inputSchema": { + "type": "object", + "properties": { + "limit": { + "type": "integer" + }, + "offset": { + "type": "integer" + } + }, + "required": [], + "additionalProperties": false + } + }, + { + "name": "task_list_worklog", + "description": "List task worklog entries with aggregate effort totals.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "limit": { + "type": [ + "integer", + "null" + ] + }, + "offset": { + "type": [ + "integer", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "List task worklog entries." + } + }, + { + "name": "task_search", + "description": "Search Von's internal task store with rich filters (status, assignee, creator, report-to, labels, planning metadata, category/source semantics, hierarchy, date ranges, and dependency state).", + "inputSchema": { + "type": "object", + "properties": { + "project_concept_id": { + "type": [ + "string", + "null" + ] + }, + "collection_concept_id": { + "type": [ + "string", + "null" + ] + }, + "query": { + "type": [ + "string", + "null" + ] + }, + "status_filter": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": [ + "string", + "null" + ] + }, + "statuses": { + "type": [ + "array", + "null" + ], + "items": {} + }, + "task_type_ids": { + "type": [ + "array", + "string", + "null" + ], + "items": {} + }, + "task_type_id": { + "type": [ + "array", + "string", + "null" + ], + "items": {} + }, + "task_source_id": { + "type": [ + "string", + "null" + ] + }, + "task_source_ids": { + "type": [ + "array", + "string", + "null" + ], + "items": {} + }, + "source_id": { + "type": [ + "string", + "null" + ] + }, + "assignee_concept_id": { + "type": [ + "string", + "null" + ] + }, + "assignee_id": { + "type": [ + "string", + "null" + ] + }, + "user_concept_id": { + "type": [ + "string", + "null" + ] + }, + "created_by_concept_id": { + "type": [ + "string", + "null" + ] + }, + "creator_concept_id": { + "type": [ + "string", + "null" + ] + }, + "report_to_concept_id": { + "type": [ + "string", + "null" + ] + }, + "reports_to_concept_id": { + "type": [ + "string", + "null" + ] + }, + "labels": { + "type": [ + "array", + "null" + ], + "items": {} + }, + "components": { + "type": [ + "array", + "null" + ], + "items": {} + }, + "fix_versions": { + "type": [ + "array", + "null" + ], + "items": {} + }, + "fixVersions": { + "type": [ + "array", + "null" + ], + "items": {} + }, + "sprint_values": { + "type": [ + "array", + "null" + ], + "items": {} + }, + "sprints": { + "type": [ + "array", + "null" + ], + "items": {} + }, + "backlog_rank": { + "type": [ + "string", + "null" + ] + }, + "parent_task_concept_id": { + "type": [ + "string", + "null" + ] + }, + "epic_task_concept_id": { + "type": [ + "string", + "null" + ] + }, + "has_parent": { + "type": [ + "boolean", + "null" + ] + }, + "has_subtasks": { + "type": [ + "boolean", + "null" + ] + }, + "has_epic": { + "type": [ + "boolean", + "null" + ] + }, + "has_backlog_rank": { + "type": [ + "boolean", + "null" + ] + }, + "start_from": { + "type": [ + "string", + "null" + ] + }, + "start_to": { + "type": [ + "string", + "null" + ] + }, + "due_from": { + "type": [ + "string", + "null" + ] + }, + "due_to": { + "type": [ + "string", + "null" + ] + }, + "created_from": { + "type": [ + "string", + "null" + ] + }, + "created_to": { + "type": [ + "string", + "null" + ] + }, + "updated_from": { + "type": [ + "string", + "null" + ] + }, + "updated_to": { + "type": [ + "string", + "null" + ] + }, + "dependency_state": { + "type": [ + "string", + "null" + ] + }, + "organisation_concept_id": { + "type": [ + "string", + "null" + ] + }, + "bulk_visibility": { + "type": [ + "string", + "null" + ] + }, + "bulk_task_visibility": { + "type": [ + "string", + "null" + ] + }, + "bulk_collection_id": { + "type": [ + "string", + "array", + "null" + ], + "items": {} + }, + "bulk_collection_ids": { + "type": [ + "string", + "array", + "null" + ], + "items": {} + }, + "limit": { + "type": [ + "integer", + "null" + ] + }, + "offset": { + "type": [ + "integer", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "Search Von internal tasks with rich filtering." + } + }, + { + "name": "task_set_parent", + "description": "Set or clear a task's parent. Supports parent reassignment and hierarchy cycle protection.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "parent_task_concept_id": { + "type": [ + "string", + "null" + ] + }, + "actor_concept_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "Set or clear parent task relationship for hierarchy management.", + "x-von-scalar-source-fields": { + "task_concept_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ], + "parent_task_concept_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ], + "task_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ] + } + } + }, + { + "name": "task_transition", + "description": "Apply a status transition to a task (Jira-style transition operation). Use task_get_transitions first for valid transition IDs.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "transition_id": { + "type": [ + "string", + "null" + ] + }, + "to_status": { + "type": [ + "string", + "null" + ] + }, + "actor_concept_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + } + }, + "required": [], + "additionalProperties": true, + "description": "Transition a task by transition_id or directly by to_status.", + "x-von-scalar-source-fields": { + "task_concept_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ], + "task_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ] + } + } + }, + { + "name": "task_unlink", + "description": "Remove typed task dependency links.", + "inputSchema": { + "type": "object", + "properties": { + "source_task_concept_id": { + "type": "string" + }, + "target_task_concept_id": { + "type": "string" + }, + "link_type": { + "type": "string" + }, + "source_task_id": { + "type": [ + "string", + "null" + ] + }, + "target_task_id": { + "type": [ + "string", + "null" + ] + }, + "actor_concept_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "source_task_concept_id", + "target_task_concept_id", + "link_type" + ], + "additionalProperties": true, + "description": "Remove a typed dependency link between two tasks." + } + }, + { + "name": "task_update_fields", + "description": "Revise an existing task and its next checkpoint, evidence, or notes. Attach the canonical current work product with current_work_product_concept_id so it opens from Tasks; its readable body belongs in the product's actor-effective hasContent. For a continuing responsibility delegated to Von, the creator can assign it to #V#von_system and set report_to_concept_id to themselves. This preserves the task and does not launch execution or a schedule. Ordinary chat edits only tasks created by or assigned to the authenticated actor in the same organisation.", + "inputSchema": { + "type": "object", + "properties": { + "task_concept_id": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "fields": { + "type": "object", + "additionalProperties": true + }, + "actor_concept_id": { + "type": [ + "string", + "null" + ] + }, + "namespace": { + "type": [ + "string", + "null" + ] + }, + "acting_user_concept_id": { + "type": [ + "string", + "null" + ] + }, + "organisation_concept_id": { + "type": [ + "string", + "null" + ] + }, + "actor_bound_continuation": { + "type": "boolean" + } + }, + "required": [], + "additionalProperties": true, + "description": "Update multiple task fields in one operation. Supported fields: status, assignee_concept_id, created_by_concept_id, title, description, priority, task_type_ids, task_source_id, report_to_concept_id, task_role, next_checkpoint, progress_signal, evidence, notes, reference_code, start_date, due_date, labels, components, fix_versions, sprint_values, backlog_rank, reporter_concept_id, watcher_concept_ids, parent_task_concept_id, epic_task_concept_id, current_work_product_concept_id. For ordinary continuation use title, description, status, priority, task_type_ids, task_role, next_checkpoint, progress_signal, evidence, notes, start_date, due_date, current_work_product_concept_id, assignee_concept_id (self or #V#von_system, creator only), and report_to_concept_id (self).", + "x-von-scalar-source-fields": { + "task_concept_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ], + "task_id": [ + "task_concept_id", + "concept_id", + "task_id", + "id" + ] + } + } + }, { "name": "testing_cleanup_arxiv_paper_ingestion_artifacts", "description": "Clean up transient artefacts created by the arXiv paper-ingestion testing workflow.", diff --git a/src/backend/server/routes/task_routes.py b/src/backend/server/routes/task_routes.py index 5336ffea5..4b8eae2d4 100644 --- a/src/backend/server/routes/task_routes.py +++ b/src/backend/server/routes/task_routes.py @@ -36,6 +36,7 @@ TaskNotFoundError, TaskOrganisationScopeAccessError, add_task_attachment, + add_task_attachment_bytes, add_task_comment, apply_bulk_task_visibility, backfill_jira_migration_bulk_task_collections, @@ -48,6 +49,8 @@ link_tasks, list_task_attachments, list_task_comments, + list_task_worklog, + add_task_worklog, list_tasks_with_visibility, search_tasks, unlink_tasks, @@ -327,6 +330,14 @@ def create_task_route() -> ResponseReturnValue: # Get creator from session creator_concept_id = _get_current_user_concept_id() organisation_concept_id = _get_current_org_concept_id() + if data.get("project_concept_id"): + from ...services.task_project_service import get_task_project + + try: + project = get_task_project(data["project_concept_id"]) + except ValueError as exc: + raise InvalidTaskDataError(str(exc)) from exc + organisation_concept_id = project.get("organisation_concept_id") parsed_start_date = _parse_optional_datetime( data.get("start_date"), "start_date" ) @@ -359,6 +370,8 @@ def create_task_route() -> ResponseReturnValue: evidence=data.get("evidence"), notes=data.get("notes"), reference_code=data.get("reference_code"), + project_concept_id=data.get("project_concept_id"), + collection_concept_ids=data.get("collection_concept_ids"), ) return jsonify(result), 201 @@ -732,6 +745,8 @@ def search_tasks_route() -> ResponseReturnValue: result = search_tasks( query=request.args.get("query"), + project_concept_id=request.args.get("project_concept_id"), + collection_concept_id=request.args.get("collection_concept_id"), status_filter=request.args.get("status_filter"), statuses=statuses or None, task_type_ids=( @@ -779,8 +794,12 @@ def search_tasks_route() -> ResponseReturnValue: updated_from=request.args.get("updated_from"), updated_to=request.args.get("updated_to"), dependency_state=request.args.get("dependency_state"), - organisation_concept_id=request.args.get("organisation_concept_id") - or _get_current_org_concept_id(), + organisation_concept_id=( + None + if request.args.get("project_concept_id") + else request.args.get("organisation_concept_id") + or _get_current_org_concept_id() + ), bulk_visibility=request.args.get("bulk_visibility") or request.args.get("bulk_task_visibility"), bulk_collection_ids=_parse_bulk_collection_ids(), @@ -798,6 +817,42 @@ def search_tasks_route() -> ResponseReturnValue: return jsonify({"error": "Internal server error"}), 500 +@task_bp.route("/projects", methods=["GET"]) +def task_projects_route() -> ResponseReturnValue: + from ...services.task_project_service import list_task_projects + + projects = list_task_projects( + limit=_parse_int_param(request.args.get("limit"), 100), + offset=_parse_int_param(request.args.get("offset"), 0), + ) + return jsonify({"projects": projects}) + + +@task_bp.route("/projects/", methods=["GET"]) +def task_project_route(project_id) -> ResponseReturnValue: + from ...services.task_project_service import ( + get_task_project, + list_project_collections, + ) + + try: + project = get_task_project(project_id) + collections = list_project_collections(project) + return jsonify({"project": project, "collections": collections}) + except ValueError: + return jsonify({"error": "Project not found"}), 404 + + +@task_bp.route("/collections/", methods=["GET"]) +def task_collection_route(collection_id) -> ResponseReturnValue: + from ...services.task_project_service import get_task_collection + + try: + return jsonify({"collection": get_task_collection(collection_id)}) + except ValueError: + return jsonify({"error": "Collection not found"}), 404 + + @task_bp.route("/my", methods=["GET"]) def my_tasks_route() -> ResponseReturnValue: """List tasks assigned to the current user. @@ -1009,6 +1064,24 @@ def list_task_attachments_route(task_concept_id: str) -> ResponseReturnValue: def add_task_attachment_route(task_concept_id: str) -> ResponseReturnValue: """Add an attachment record to a task.""" try: + if request.content_type and request.content_type.startswith( + "multipart/form-data" + ): + actor_id = _get_current_user_concept_id() + if not actor_id: + return jsonify({"error": "Not authenticated"}), 401 + upload = request.files.get("file") + if upload is None: + raise InvalidTaskDataError("file is required") + attachment = add_task_attachment_bytes( + task_concept_id, + data=upload.stream.read(100 * 1024 * 1024 + 1), + filename=upload.filename, + media_type=upload.mimetype, + actor_concept_id=actor_id, + note=request.form.get("note"), + ) + return jsonify(attachment), 201 data = request.get_json() or {} filename = data.get("filename") uri = data.get("uri") @@ -1071,6 +1144,62 @@ def get_task_history_route(task_concept_id: str) -> ResponseReturnValue: return jsonify({"error": "Internal server error"}), 500 +@task_bp.route("//source", methods=["GET"]) +def get_task_source_route(task_concept_id: str) -> ResponseReturnValue: + from ...services.jira_source_retention_service import get_retained_task_source + + actor = _get_current_user_concept_id() + if not actor: + return jsonify({"error": "Authentication required"}), 401 + try: + result = get_retained_task_source( + task_concept_id, + actor_concept_id=actor, + resource=request.args.get("resource"), + offset=_parse_int_param(request.args.get("offset"), 0), + limit=_parse_int_param(request.args.get("limit"), 100), + ) + return jsonify(result), 200 + except (ValueError, TaskNotFoundError) as exc: + return jsonify({"error": str(exc)}), 404 + + +@task_bp.route("//worklog", methods=["GET", "POST"]) +def task_worklog_route(task_concept_id: str) -> ResponseReturnValue: + try: + if request.method == "GET": + return ( + jsonify( + list_task_worklog( + task_concept_id, + limit=_parse_int_param(request.args.get("limit"), 100), + offset=_parse_int_param(request.args.get("offset"), 0), + ) + ), + 200, + ) + actor = _get_current_user_concept_id() + if not actor: + return jsonify({"error": "Authentication required"}), 401 + data = request.get_json() or {} + return ( + jsonify( + add_task_worklog( + task_concept_id, + author_concept_id=actor, + time_spent_minutes=data.get("time_spent_minutes"), + comment=data.get("comment"), + started_at=data.get("started_at"), + ) + ), + 201, + ) + except TaskNotFoundError as exc: + return jsonify({"error": str(exc)}), 404 + except InvalidTaskDataError as exc: + return jsonify({"error": str(exc)}), 400 + + @task_bp.route("//links", methods=["POST"]) def add_task_link_route(task_concept_id: str) -> ResponseReturnValue: """Add a typed link between tasks.""" diff --git a/src/backend/services/computer_file_copy_service.py b/src/backend/services/computer_file_copy_service.py index 0316446e6..b705a6b1e 100644 --- a/src/backend/services/computer_file_copy_service.py +++ b/src/backend/services/computer_file_copy_service.py @@ -434,6 +434,7 @@ def create_computer_file_copy_instance( blob_uri: str, visibility_scope_mode: str | None = None, metadata_in_attributes: bool = False, + maintain_relationship_inverses: bool = True, metadata: Mapping[str, Any] | None = None, logger: Any | None = None, ) -> ComputerFileCopyRecord: @@ -528,6 +529,7 @@ def create_computer_file_copy_instance( organisation_concept_id=effective_organisation_concept_id, event_namespace=canonical_namespace, visibility_scope_mode=visibility_scope_mode, + maintain_relationship_inverses=maintain_relationship_inverses, ) if metadata_in_attributes: @@ -1193,6 +1195,10 @@ def import_bytes_file_copy( source_uri: str | None = None, blob_key: str | None = None, metadata: Mapping[str, Any] | None = None, + metadata_in_attributes: bool = False, + infer_typing: bool = True, + maintain_relationship_inverses: bool = True, + visibility_scope_mode: str | None = None, ) -> dict[str, Any]: """Persist bytes durably and register a blob-backed file-copy concept. @@ -1349,6 +1355,9 @@ def import_bytes_file_copy( blob_key=stored.ref.key, blob_uri=stored.ref.uri, metadata=persisted_metadata, + metadata_in_attributes=metadata_in_attributes, + maintain_relationship_inverses=maintain_relationship_inverses, + visibility_scope_mode=visibility_scope_mode, ) except Exception as exc: return { @@ -1375,14 +1384,26 @@ def import_bytes_file_copy( persist_file_copy_typing, ) - typing_result = infer_file_copy_typing( - content_type=content_type, - original_filename=original_filename, - size_bytes=size_bytes, + typing_result = ( + infer_file_copy_typing( + content_type=content_type, + original_filename=original_filename, + size_bytes=size_bytes, + ) + if infer_typing + else None ) - typing_persist_result = persist_file_copy_typing( - file_copy_concept_id=created.concept_id, - typing_result=typing_result, + typing_persist_result = ( + persist_file_copy_typing( + file_copy_concept_id=created.concept_id, + typing_result=typing_result, + ) + if infer_typing + else { + "success": True, + "typing_persisted": False, + "status": "declared_file_type", + } ) except Exception as exc: typing_persist_result = { diff --git a/src/backend/services/jira_source_retention_service.py b/src/backend/services/jira_source_retention_service.py new file mode 100644 index 000000000..d56075af8 --- /dev/null +++ b/src/backend/services/jira_source_retention_service.py @@ -0,0 +1,609 @@ +"""Lossless source capture for the existing Jira task importer. + +Operational task projections are deliberately separate from immutable source +copies. JSON values (including ADF) and original pages are preserved; attachment +bytes are retained with SHA-256 hashes. Manifests never infer completeness from +an issue key, HTTP success, or an embedded first page. +""" + +from __future__ import annotations + +import asyncio +import base64 +import copy +import gzip +import hashlib +import json +from datetime import UTC, datetime +from typing import Any + + +def canonical_json(value: Any) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _source_payload(value): + # The proxy adds its receipt; it is provenance, not part of the Jira source. + if isinstance(value, dict): + return {key: item for key, item in value.items() if key != "authority"} + return value + + +async def capture_resource( + proxy, resource, identifier="", secondary_id="", *, item_key=None +): + """Retain every page, rejecting stalled, inconsistent or duplicate paging.""" + pages, items, errors, seen = [], [], [], set() + start_at, expected_total = 0, None + next_page_token, seen_tokens = None, set() + while True: + try: + page = _source_payload( + await proxy.get_migration_resource( + resource=resource, + identifier=str(identifier), + secondary_id=str(secondary_id), + start_at=start_at, + max_results=100, + **( + {"next_page_token": next_page_token} + if resource in {"board_issues", "filter_issues"} + else {} + ), + ) + ) + except Exception as exc: # noqa: BLE001 - Retain partial evidence. + errors.append({"error": type(exc).__name__, "message": str(exc)}) + break + pages.append(page) + if isinstance(page, dict) and page.get("success") is False: + errors.append( + {"error": page.get("error"), "status_code": page.get("status_code")} + ) + break + if item_key is None: + if isinstance(page, list): + items.extend(page) + break + values = page.get(item_key) if isinstance(page, dict) else None + if not isinstance(values, list): + errors.append({"error": "missing_page_items", "item_key": item_key}) + break + if page.get("startAt", start_at) != start_at: + errors.append({"error": "unexpected_page_offset"}) + break + total = page.get("total") + if isinstance(total, int): + if expected_total is not None and total != expected_total: + errors.append({"error": "source_changed_during_paging"}) + expected_total = total + for item in values: + identity = ( + str(item.get("id")) + if isinstance(item, dict) and "id" in item + else canonical_json(item).hex() + ) + if identity in seen: + errors.append({"error": "duplicate_source_item", "id": identity}) + seen.add(identity) + items.extend(values) + next_offset = start_at + len(values) + if page.get("isLast") is True or ( + expected_total is not None and next_offset >= expected_total + ): + if expected_total is not None and len(items) != expected_total: + errors.append( + { + "error": "source_count_mismatch", + "expected": expected_total, + "actual": len(items), + } + ) + break + if not values: + errors.append({"error": "incomplete_or_unproven_pagination"}) + break + if resource in {"board_issues", "filter_issues"}: + token = page.get("nextPageToken") + if not token or token in seen_tokens: + errors.append({"error": "missing_or_repeated_page_token"}) + break + seen_tokens.add(token) + next_page_token = token + # A source must expose a terminal indicator, not merely a short page. + if expected_total is None and "isLast" not in page and not next_page_token: + errors.append({"error": "missing_pagination_contract"}) + break + if errors: + break + start_at = next_offset + return { + "resource": resource, + "identifier": str(identifier), + "secondary_id": str(secondary_id), + "pages": pages, + "items": items, + "count": len(items), + "complete": not errors, + "errors": errors, + } + + +async def capture_issue( + proxy, issue_key: str, *, attachment_max_size_bytes: int = 100 * 1024 * 1024 +): + """Return an unchanged source archive and a separate hydrated projection.""" + original = _source_payload( + await proxy.get_issue( + issue_key=issue_key, + fields=["*all"], + expand=["names", "schema", "renderedFields"], + ) + ) + if ( + not isinstance(original, dict) + or original.get("success") is False + or not isinstance(original.get("fields"), dict) + ): + raise ValueError(f"Jira issue capture failed: {issue_key}") + if str(original.get("key", "")).upper() != issue_key.upper(): + raise ValueError("Jira issue identity mismatch") + hydrated = copy.deepcopy(original) + # Some Jira subresource routes fail for a key that get_issue still resolves. + # Use the captured stable ID for the same issue and retain it in each page. + issue_identifier = str(original.get("id") or issue_key) + # Independent source pages can be read together. Limit concurrency to four + # to bound Jira load; ordering and the final source-version check are stable. + requests = [ + ("comments", "comments"), + ("worklogs", "worklogs"), + ("changelog", "values"), + ("remote_links", None), + ("issue_properties", None), + ("votes", None), + ] + resources = {} + for offset in range(0, len(requests), 4): + batch = requests[offset : offset + 4] + results = await asyncio.gather( + *( + capture_resource(proxy, name, issue_identifier, item_key=item_key) + for name, item_key in batch + ) + ) + resources.update({name: result for (name, _), result in zip(batch, results)}) + for resource, item_key, destination in ( + ("comments", "comments", "comment"), + ("worklogs", "worklogs", "worklog"), + ("changelog", "values", None), + ): + captured = resources[resource] + if destination: + hydrated["fields"][destination] = { + item_key: captured["items"], + "total": captured["count"], + "startAt": 0, + } + else: + hydrated["changelog"] = { + "histories": captured["items"], + "total": captured["count"], + "startAt": 0, + } + property_pages = resources["issue_properties"]["pages"] + properties = ( + property_pages[0].get("keys", []) + if property_pages and isinstance(property_pages[0], dict) + else [] + ) + for prop in properties: + key = prop.get("key") + if isinstance(key, str): + resources[f"issue_property:{key}"] = await capture_resource( + proxy, "issue_property", issue_identifier, key + ) + try: + watchers = _source_payload(await proxy.get_watchers(issue_key=issue_key)) + ok = isinstance(watchers, dict) and watchers.get("success") is not False + if ok and watchers.get("watchCount") is not None: + ok = watchers["watchCount"] == len(watchers.get("watchers", [])) + resources["watchers"] = { + "pages": [watchers], + "complete": ok, + "errors": [] if ok else [{"error": "watchers_incomplete"}], + } + hydrated["watchers"] = watchers + except Exception as exc: # noqa: BLE001 - Retain partial evidence. + resources["watchers"] = { + "complete": False, + "errors": [{"error": type(exc).__name__}], + } + binaries = [] + for attachment in hydrated["fields"].get("attachment") or []: + attachment_id = str(attachment.get("id", "")) + record = {"id": attachment_id, "complete": False} + try: + result = _source_payload( + await proxy.get_attachment_content( + attachment_id=attachment_id, + max_size_bytes=attachment_max_size_bytes, + ) + ) + record["source"] = result + if result.get("success") is not True: + raise ValueError(result.get("error", "attachment_fetch_failed")) + data = base64.b64decode(result["content_base64"], validate=True) + if len(data) != int( + attachment.get("size", result.get("size_bytes", len(data))) + ): + raise ValueError("attachment_size_mismatch") + record.update( + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + complete=True, + ) + attachment["content_base64"] = result["content_base64"] + except Exception as exc: # noqa: BLE001 - Retain partial evidence. + record["error"] = str(exc) + binaries.append(record) + after = await proxy.get_issue(issue_key=issue_key, fields=["updated"]) + stable = after.get("fields", {}).get("updated") == original["fields"].get("updated") + manifest = { + "schema": "jira_source_archive.v1", + "kind": "issue", + "source": original, + "resources": resources, + "binaries": binaries, + "source_stable": stable, + "complete": stable + and all(x["complete"] for x in resources.values()) + and all(x["complete"] for x in binaries), + } + return manifest, hydrated + + +async def capture_project(proxy, project_key): + """Retain project data, configuration and nested properties/role memberships.""" + resources = {"project": await capture_resource(proxy, "project", project_key)} + pages = resources["project"]["pages"] + project = pages[0] if pages and isinstance(pages[0], dict) else {} + project_id = project.get("id", project_key) + names = ( + "components", + "versions", + "project_properties", + "project_roles", + "project_statuses", + "permission_scheme", + "security_scheme", + "notification_scheme", + "workflow_scheme", + "issue_type_scheme", + "field_configuration_scheme", + "screen_scheme", + ) + for offset in range(0, len(names), 4): + batch = names[offset : offset + 4] + results = await asyncio.gather( + *( + capture_resource( + proxy, + name, + project_id, + item_key=( + "values" + if name + in { + "issue_type_scheme", + "field_configuration_scheme", + "screen_scheme", + } + else None + ), + ) + for name in batch + ) + ) + resources.update(dict(zip(batch, results))) + security = resources["security_scheme"] + for page in security["pages"]: + if isinstance(page, dict) and page.get("status_code") == 404: + try: + messages = json.loads(page.get("response", "{}"))["errorMessages"] + except (ValueError, KeyError, TypeError): + messages = [] + if messages == [f"Security level for project {project_id} does not exist."]: + security.update( + complete=True, errors=[], disposition="source_reports_absent" + ) + for prop in (resources["project_properties"]["pages"] or [{}])[0].get("keys", []): + if prop.get("key"): + resources[f"property:{prop['key']}"] = await capture_resource( + proxy, "project_property", project_id, prop["key"] + ) + role_page = (resources["project_roles"]["pages"] or [{}])[0] + if isinstance(role_page, dict) and resources["project_roles"]["complete"]: + for role_name, url in role_page.items(): + if isinstance(url, str) and url.rsplit("/", 1)[-1].isdigit(): + resources[f"role:{role_name}"] = await capture_resource( + proxy, "project_role", project_id, url.rsplit("/", 1)[-1] + ) + return { + "schema": "jira_source_archive.v1", + "kind": "project", + "source": project, + "resources": resources, + "complete": all(x["complete"] for x in resources.values()), + } + + +async def capture_site_configuration(proxy, *, site_url): + """Shared definitions are captured once and referenced by project records. + + Separate export surfaces are explicit outstanding dispositions. A successful + REST crawl alone cannot certify third-party app or Automation retention. + """ + resources = {} + for resource in ( + "projects", + "fields", + "issue_types", + "statuses", + "resolutions", + "priorities", + "workflows", + "screens", + "filters", + "dashboards", + "webhooks", + "boards", + ): + paginated = resource in { + "projects", + "workflows", + "screens", + "filters", + "dashboards", + "webhooks", + "boards", + } + resources[resource] = await capture_resource( + proxy, resource, item_key="values" if paginated else None + ) + for field in resources["fields"]["items"]: + if not field.get("custom"): + continue + field_id = field["id"] + contexts = await capture_resource( + proxy, "field_contexts", field_id, item_key="values" + ) + resources[f"field_contexts:{field_id}"] = contexts + resources[f"field_defaults:{field_id}"] = await capture_resource( + proxy, "field_defaults", field_id, item_key="values" + ) + schema = field.get("schema", {}) + if schema.get("type") == "option" or schema.get("items") == "option": + for context in contexts["items"]: + resources[f"field_options:{field_id}:{context['id']}"] = ( + await capture_resource( + proxy, + "field_options", + field_id, + context["id"], + item_key="values", + ) + ) + for board in resources["boards"]["items"]: + board_id = board["id"] + for resource in ( + "board_configuration", + "board_projects", + "board_issues", + "board_properties", + "board_quick_filters", + ): + resources[f"{resource}:{board_id}"] = await capture_resource( + proxy, + resource, + board_id, + item_key=( + "issues" + if resource in {"board_issues", "filter_issues"} + else ( + "values" + if resource in {"board_projects", "board_quick_filters"} + else None + ) + ), + ) + for prop in (resources[f"board_properties:{board_id}"].get("pages") or [{}])[ + 0 + ].get("keys", []): + if prop.get("key"): + resources[f"board_property:{board_id}:{prop['key']}"] = ( + await capture_resource( + proxy, "board_property", board_id, prop["key"] + ) + ) + if board.get("type") == "scrum": + resources[f"board_sprints:{board_id}"] = await capture_resource( + proxy, "board_sprints", board_id, item_key="values" + ) + for dashboard in resources["dashboards"]["items"]: + for resource in ("dashboard", "dashboard_gadgets"): + resources[f"{resource}:{dashboard['id']}"] = await capture_resource( + proxy, resource, dashboard["id"] + ) + for source_filter in resources["filters"]["items"]: + resources[f"filter:{source_filter['id']}"] = await capture_resource( + proxy, "filter", source_filter["id"] + ) + resources[f"filter_issues:{source_filter['id']}"] = await capture_resource( + proxy, "filter_issues", source_filter["id"], item_key="issues" + ) + outstanding = [ + {"resource": "automation_rules", "disposition": "separate_export_required"}, + { + "resource": "third_party_app_data_and_forms", + "disposition": "app_inventory_and_exports_required", + }, + { + "resource": "external_project_documents", + "disposition": "linked_resource_inventory_required", + }, + ] + return { + "schema": "jira_source_archive.v1", + "kind": "site", + "source": { + "self": site_url, + "id": hashlib.sha256(site_url.encode()).hexdigest()[:24], + }, + "resources": resources, + "outstanding_dispositions": outstanding, + "complete": False, + "rest_resources_complete": all( + value["complete"] for value in resources.values() + ), + } + + +def store_source_archive( + archive, *, actor_concept_id, organisation_concept_id=None, namespace=None +): + """Content-addressed immutable file copy; reruns reuse identical bytes.""" + from .computer_file_copy_service import import_bytes_file_copy + + raw = canonical_json(archive) + digest = hashlib.sha256(raw).hexdigest() + data = gzip.compress(raw, mtime=0) + source = archive.get("source", {}) + source_id = str(source.get("id") or source.get("key") or "site") + kind = archive["kind"] + scope = hashlib.sha256( + f"{actor_concept_id}\n{organisation_concept_id}".encode() + ).hexdigest()[:24] + from ..security.access_control import override_current_actor + + # Use the archive's explicit audience context during registration; a + # selected operational project must not leak its ambient organisation here. + with override_current_actor(actor_concept_id, organisation_concept_id): + receipt = import_bytes_file_copy( + data=data, + user_concept_id=actor_concept_id, + organisation_concept_id=organisation_concept_id, + namespace=namespace, + original_filename=f"jira-{kind}-{source_id}-{digest[:12]}.json.gz", + content_type="application/gzip", + source_system="jira_source_archive", + source_identifier=source_id, + source_uri=source.get("self"), + blob_key=f"jira-source/{scope}/{kind}/{source_id}/{digest}.json.gz", + metadata={ + "schema": archive["schema"], + "content_sha256": digest, + "source_id": source_id, + }, + metadata_in_attributes=True, + infer_typing=False, + maintain_relationship_inverses=False, + visibility_scope_mode=( + "user_only_default" + if not organisation_concept_id + else "user_org_default" + ), + ) + if receipt.get("success") is not True: + raise RuntimeError(f"Source archive persistence failed: {receipt.get('error')}") + return { + "file_copy_concept_id": receipt["concept_id"], + "content_sha256": digest, + "blob_sha256": hashlib.sha256(data).hexdigest(), + "size_bytes": len(data), + "complete": bool(archive["complete"]), + "schema": archive["schema"], + "captured_at": datetime.now(UTC).isoformat(), + } + + +def read_source_archive( + reference, *, actor_concept_id, organisation_concept_id=None, namespace=None +): + """Retrieve through file-copy access checks and verify both digest layers.""" + from .computer_file_copy_service import fetch_file_copy_bytes + + result = fetch_file_copy_bytes( + file_copy_concept_id=reference["file_copy_concept_id"], + user_concept_id=actor_concept_id, + organisation_concept_id=organisation_concept_id, + namespace=namespace, + allow_large=True, + ) + # The file-copy service also checks its registered stored-byte digest. + if result.get("success") is not True: + raise ValueError("Jira archive is unavailable to this actor") + data = result["data"] + if hashlib.sha256(data).hexdigest() != reference["blob_sha256"]: + raise ValueError("Jira archive blob checksum mismatch") + raw = gzip.decompress(data) + if hashlib.sha256(raw).hexdigest() != reference["content_sha256"]: + raise ValueError("Jira archive content checksum mismatch") + return json.loads(raw) + + +def get_retained_task_source( + task_concept_id, *, actor_concept_id, resource=None, offset=0, limit=100 +): + """Read preserved source through the task and file-copy access boundaries. + + Binary bodies remain in the downloadable original; paged activity can be + inspected without returning every attachment as base64 in a tool response. + """ + from urllib.parse import quote + + from .task_management_service import get_task + + task = get_task(task_concept_id) + reference = ( + task.get("external_references", {}).get("jira", {}).get("source_archive") + ) + if not reference: + raise ValueError("No retained Jira source archive for this task") + archive = read_source_archive(reference, actor_concept_id=actor_concept_id) + result = { + "task_concept_id": task_concept_id, + "reference": reference, + "download_url": f"/von/api/files/{quote(reference['file_copy_concept_id'], safe='')}/download", + "complete": archive["complete"], + "source_stable": archive.get("source_stable"), + "resources": { + key: {k: v for k, v in value.items() if k not in {"pages", "items"}} + for key, value in archive.get("resources", {}).items() + }, + "binaries": [ + {k: v for k, v in item.items() if k != "source"} + for item in archive.get("binaries", []) + ], + } + if resource: + captured = archive.get("resources", {}).get(resource) + if captured is None: + raise ValueError("Unknown retained source resource") + values = captured.get("items") or captured.get("pages", []) + offset, limit = max(0, int(offset)), max(1, min(200, int(limit))) + result.update( + resource=resource, + items=values[offset : offset + limit], + total=len(values), + offset=offset, + next_offset=offset + limit if offset + limit < len(values) else None, + ) + else: + result["source"] = archive["source"] + return result diff --git a/src/backend/services/jira_task_import_service.py b/src/backend/services/jira_task_import_service.py index cab74395e..0602ac24b 100644 --- a/src/backend/services/jira_task_import_service.py +++ b/src/backend/services/jira_task_import_service.py @@ -17,6 +17,7 @@ import uuid from datetime import datetime, timezone from typing import Any, Dict, Mapping, Sequence +from urllib.parse import urlsplit from ..db.repositories.concepts_repository import ConceptsRepository from ..db.repositories.text_value_repository import ( @@ -26,7 +27,12 @@ from ..utils.jira_issue_key_utils import normalise_jira_issue_key from ..utils.concept_id_utils import canonicalise_vontology_concept_id, ensure_v_concept_prefix from .concept_resolution_service import resolve_concept_by_name -from .concept_service import create_concept, get_concept_by_concept_id +from .concept_service import ( + create_concept, + get_concept_by_concept_id, + get_concept_by_concept_id_exact, + ConceptNotFoundError, +) from .text_value_service import upsert_text_for_concept from .task_management_service import ( InvalidTaskDataError, @@ -39,6 +45,7 @@ build_jira_migration_bulk_task_collection, create_task, find_task_by_external_reference, + get_task, link_tasks, record_task_history_event, set_task_epic, @@ -47,6 +54,7 @@ upsert_task_external_reference, ) from .task_ontology_service import JIRA_IMPORTED_TASK_SOURCE_ID +from .feature_flags import suppress_event_workflow_launches _normalise_issue_key = normalise_jira_issue_key @@ -168,13 +176,22 @@ def _extract_priority_name(fields: Mapping[str, Any]) -> str | None: return None -def _map_status(status_name: str | None) -> tuple[str, str | None]: +def _map_status( + status_name: str | None, category_key: str | None = None +) -> tuple[str, str | None]: if not isinstance(status_name, str) or not status_name.strip(): return _DEFAULT_STATUS, "jira_status_missing_defaulted_to_pending" normalised = status_name.strip().lower() mapped = _JIRA_STATUS_TO_VON_STATUS.get(normalised) if mapped: return mapped, None + category_status = { + "new": "pending", + "indeterminate": "in_progress", + "done": "completed", + }.get(category_key) + if category_status: + return category_status, None return _DEFAULT_STATUS, f"jira_status_unmapped:{status_name}" @@ -833,6 +850,7 @@ def _ensure_jira_participant_concept( organisation_concept_id: str | None, allow_lookup: bool, allow_create: bool, + allow_alias_writes: bool = True, ) -> Dict[str, Any]: participant = _extract_jira_participant(raw_participant) if participant is None: @@ -863,6 +881,21 @@ def _ensure_jira_participant_concept( mapped_concept_id = cached_email resolution = "mapped_from_email_cache" + if mapped_concept_id is None and allow_lookup and not allow_alias_writes: + candidate = _build_jira_person_concept_id( + account_id=account_id, + email_address=email_address, + display_name=display_name, + ) + try: + existing = get_concept_by_concept_id_exact(candidate) + if existing: + mapped_concept_id, resolution = ( + existing["concept_id"], + "mapped_existing_candidate_concept", + ) + except ConceptNotFoundError: + pass if mapped_concept_id is None and allow_lookup: resolved = _resolve_existing_person_concept_id( account_id=account_id if isinstance(account_id, str) else None, @@ -901,6 +934,14 @@ def _ensure_jira_participant_concept( create_as_instance=True, created_by_concept_id=actor_concept_id, organisation_concept_id=organisation_concept_id, + **( + { + "visibility_scope_mode": "user_only_default", + "maintain_relationship_inverses": False, + } + if not allow_alias_writes + else {} + ), ) created_concept_id = created.get("concept_id") if isinstance(created, Mapping) else None if isinstance(created_concept_id, str) and created_concept_id.strip(): @@ -910,7 +951,10 @@ def _ensure_jira_participant_concept( created_concept = True resolution = "created_new_concept" except Exception: - existing = get_concept_by_concept_id(candidate_concept_id) + try: + existing = get_concept_by_concept_id_exact(candidate_concept_id) + except ConceptNotFoundError: + existing = None existing_concept_id = existing.get("concept_id") if isinstance(existing, Mapping) else None if isinstance(existing_concept_id, str) and existing_concept_id.strip(): mapped_concept_id = existing_concept_id.strip() @@ -919,13 +963,16 @@ def _ensure_jira_participant_concept( if isinstance(mapped_concept_id, str) and mapped_concept_id: mapped_concept_id = ensure_v_concept_prefix(mapped_concept_id) or mapped_concept_id try: - _upsert_jira_participant_aliases( - concept_id=mapped_concept_id, - account_id=account_id if isinstance(account_id, str) else None, - email_address=( - email_address if isinstance(email_address, str) and email_address else None - ), - ) + if allow_alias_writes and resolution != "mapped_from_account_cache": + _upsert_jira_participant_aliases( + concept_id=mapped_concept_id, + account_id=account_id if isinstance(account_id, str) else None, + email_address=( + email_address + if isinstance(email_address, str) and email_address + else None + ), + ) except Exception: # Alias persistence is best-effort; mapping should still proceed. pass @@ -1010,6 +1057,8 @@ def _issue_reference_payload( "status_history": status_history, "imported_at": _iso_now(), } + if isinstance(issue.get("_von_source_archive"), Mapping): + payload["source_archive"] = dict(issue["_von_source_archive"]) assignee_payload = _participant_reference_payload( participant=assignee, @@ -1056,6 +1105,35 @@ def _issue_reference_payload( return payload +def _legacy_import_projection(reference): + """Recover only projections actually recorded by earlier importer releases.""" + baseline = {} + for source, target in ( + ("labels", "labels"), + ("components", "components"), + ("fix_versions", "fix_versions"), + ("sprint_values", "sprint_values"), + ("backlog_rank", "backlog_rank"), + ("due_date", "due_date"), + (_JIRA_PARTICIPANT_REPORTER_METADATA_KEY, "reporter_concept_id"), + (_JIRA_PARTICIPANT_WATCHERS_METADATA_KEY, "watcher_concept_ids"), + ): + if source in reference: + baseline[target] = reference[source] + if "status_name" in reference: + baseline["status"] = _map_status(reference["status_name"])[0] + if "priority_name" in reference: + baseline["priority"] = _map_priority(reference["priority_name"])[0] + for source, target in ( + ("assignee", "assignee_concept_id"), + ("creator", "created_by_concept_id"), + ): + participant = reference.get(source) + if isinstance(participant, Mapping) and "concept_id" in participant: + baseline[target] = participant["concept_id"] + return baseline + + def _existing_imported_activity_checkpoint( existing_task: Mapping[str, Any] | None, ) -> dict[str, set[str]]: @@ -1150,6 +1228,7 @@ def _resolve_participant(raw_participant: Mapping[str, Any] | None) -> dict[str, actor_concept_id=actor_concept_id, organisation_concept_id=organisation_concept_id, allow_lookup=bool(auto_resolve_participants), + allow_alias_writes=not dry_run and not raw_issue.get("_von_source_archive"), allow_create=( bool(auto_resolve_participants) and bool(create_missing_participant_concepts) @@ -1514,6 +1593,44 @@ def _resolve_participant(raw_participant: Mapping[str, Any] | None) -> dict[str, return imported_activity, mapped_fields, dropped_fields +def _projection_value(field_name, value): + """Compare stored ISO dates and source datetimes without spurious conflicts.""" + if field_name in {"due_date", "start_date"} and value: + parsed = ( + value + if isinstance(value, datetime) + else datetime.fromisoformat(str(value).replace("Z", "+00:00")) + ) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc).isoformat() + return value + + +def _find_task_by_jira_source_identity(issue, organisation_concept_id): + """A moved/renamed Jira key must keep the original task identity.""" + issue_id = issue.get("id") + if not issue_id: + return None + query = { + "relationships.is_an_instance_of": TASK_SPECIFICATION_TYPE_ID, + "metadata.external_references.jira.issue_id": str(issue_id), + } + site = urlsplit(issue.get("self") or "").netloc + if site: + query["metadata.external_references.jira.issue_url"] = { + "$regex": r"^https?://" + re.escape(site) + "/" + } + if organisation_concept_id: + query["metadata.organisation_concept_id"] = { + "$in": [organisation_concept_id, None] + } + docs = list(ConceptsRepository.find(query, projection={"concept_id": 1}, limit=2)) + if len(docs) > 1: + raise InvalidTaskDataError("Duplicate tasks for the same Jira source issue ID") + return get_task(docs[0]["concept_id"]) if docs else None + + def _resolve_existing_task_id( *, jira_issue_key: str, @@ -1866,6 +1983,7 @@ def _build_pilot_validation_report( } +@suppress_event_workflow_launches("jira_source_import") def import_jira_issues_to_tasks( *, issues: Sequence[Mapping[str, Any]], @@ -1921,6 +2039,7 @@ def import_jira_issues_to_tasks( issue_results: list[Dict[str, Any]] = [] existing_cache: dict[str, str | None] = {} + project_writer_cache = {} task_id_by_issue_key: dict[str, str] = {} project_observations: dict[str, Dict[str, Any]] = {} @@ -2001,7 +2120,10 @@ def import_jira_issues_to_tasks( ) jira_status_name = _extract_status_name(fields) - mapped_status, status_warning = _map_status(jira_status_name) + mapped_status, status_warning = _map_status( + jira_status_name, + (fields.get("status") or {}).get("statusCategory", {}).get("key"), + ) if jira_status_name: project_observation["status_names"].add(jira_status_name) if status_warning and status_warning.startswith("jira_status_unmapped"): @@ -2061,6 +2183,7 @@ def import_jira_issues_to_tasks( actor_concept_id=actor_concept_id, organisation_concept_id=organisation_concept_id, allow_lookup=bool(auto_resolve_participants), + allow_alias_writes=not dry_run and not raw_issue.get("_von_source_archive"), allow_create=( bool(auto_resolve_participants) and bool(create_missing_participant_concepts) @@ -2075,6 +2198,7 @@ def import_jira_issues_to_tasks( actor_concept_id=actor_concept_id, organisation_concept_id=organisation_concept_id, allow_lookup=bool(auto_resolve_participants), + allow_alias_writes=not dry_run and not raw_issue.get("_von_source_archive"), allow_create=( bool(auto_resolve_participants) and bool(create_missing_participant_concepts) @@ -2089,6 +2213,7 @@ def import_jira_issues_to_tasks( actor_concept_id=actor_concept_id, organisation_concept_id=organisation_concept_id, allow_lookup=bool(auto_resolve_participants), + allow_alias_writes=not dry_run and not raw_issue.get("_von_source_archive"), allow_create=( bool(auto_resolve_participants) and bool(create_missing_participant_concepts) @@ -2123,6 +2248,8 @@ def import_jira_issues_to_tasks( actor_concept_id=actor_concept_id, organisation_concept_id=organisation_concept_id, allow_lookup=bool(auto_resolve_participants), + allow_alias_writes=not dry_run + and not raw_issue.get("_von_source_archive"), allow_create=( bool(auto_resolve_participants) and bool(create_missing_participant_concepts) @@ -2234,6 +2361,10 @@ def import_jira_issues_to_tasks( external_id=issue_key, organisation_concept_id=organisation_concept_id, ) + if not existing_task and raw_issue.get("_von_source_archive"): + existing_task = _find_task_by_jira_source_identity( + raw_issue, organisation_concept_id + ) existing_task_id = ( existing_task.get("task_concept_id") if isinstance(existing_task, Mapping) @@ -2260,6 +2391,51 @@ def import_jira_issues_to_tasks( "task_source_id": JIRA_IMPORTED_TASK_SOURCE_ID, "reference_code": issue_key, } + source_projection = None + sync_conflicts = [] + previous_reference = ( + (existing_task or {}).get("external_references", {}).get("jira", {}) + ) + # After cutover every import path (including old schedules) must honour + # the canonical project's writer, even if this call has no new archive. + writer_project_id = (existing_task or {}).get("project_concept_id") or ( + raw_issue.get("_von_project_context") or {} + ).get("project_concept_id") + if not writer_project_id: + from .task_project_service import find_jira_task_project + + cache_key = (project_key, organisation_concept_id) + if cache_key not in project_writer_cache: + project_writer_cache[cache_key] = find_jira_task_project( + project_key=project_key, + source_site=urlsplit(raw_issue.get("self") or "").netloc or None, + organisation_concept_id=organisation_concept_id, + ) + writer_project_id = (project_writer_cache[cache_key] or {}).get( + "concept_id" + ) + if writer_project_id: + from .task_project_service import get_task_project + + if get_task_project(writer_project_id).get("writer") == "von": + issue_results.append( + { + "jira_issue_key": issue_key, + "task_concept_id": existing_task_id_str, + "action": "skipped_von_authoritative", + "relation_results": [], + } + ) + task_id_by_issue_key[issue_key] = existing_task_id_str + continue + project_context = raw_issue.get("_von_project_context") + if isinstance(project_context, Mapping): + update_fields_payload.update( + { + key: project_context[key] + for key in ("project_concept_id", "collection_concept_ids") + } + ) if mark_bulk_migration_collection: update_fields_payload["bulk_task_collections"] = [ build_jira_migration_bulk_task_collection() @@ -2288,6 +2464,68 @@ def import_jira_issues_to_tasks( if rank_value: update_fields_payload["backlog_rank"] = rank_value + if raw_issue.get("_von_source_archive"): + update_fields_payload.update(title=title, description=description) + # Additional native collection membership survives source refreshes. + if existing_task and "collection_concept_ids" in update_fields_payload: + update_fields_payload["collection_concept_ids"] = sorted( + set(update_fields_payload["collection_concept_ids"]) + | set(existing_task.get("collection_concept_ids") or []) + ) + source_projection = { + key: _projection_value(key, value) + for key, value in update_fields_payload.items() + } + if ( + existing_task_id_str + and previous_reference.get("import_phase") != "initialising" + ): + baseline = previous_reference.get("last_import_projection", {}) + # Legacy imports retained some original projections. Unknown + # baselines stay conflicts; do not assume native edits are stale. + if not baseline: + baseline = _legacy_import_projection(previous_reference) + for field_name, incoming in list(update_fields_payload.items()): + if field_name in { + "project_concept_id", + "collection_concept_ids", + "bulk_task_collections", + "task_source_id", + "reference_code", + }: + continue + current = _projection_value( + field_name, existing_task.get(field_name) + ) + incoming = _projection_value(field_name, incoming) + previous = _projection_value(field_name, baseline.get(field_name)) + if current != incoming and ( + field_name not in baseline or current != previous + ): + unresolved = any( + row.get("field") == field_name + for row in previous_reference.get("sync_conflicts", []) + ) + if ( + field_name not in baseline + or incoming != previous + or unresolved + ): + sync_conflicts.append( + { + "field": field_name, + "resolution": "preserved_von_value", + } + ) + del update_fields_payload[field_name] + if existing_task: + update_fields_payload = { + key: value + for key, value in update_fields_payload.items() + if _projection_value(key, existing_task.get(key)) + != _projection_value(key, value) + } + try: if existing_task_id_str is not None: task_id = existing_task_id_str @@ -2295,11 +2533,15 @@ def import_jira_issues_to_tasks( action = "would_update" summary["would_update"] += 1 elif update_existing: - update_task_fields( - task_id, - fields=update_fields_payload, - actor_concept_id=actor_concept_id, - ) + if update_fields_payload: + update_task_fields( + task_id, + fields=update_fields_payload, + actor_concept_id=actor_concept_id, + preserve_visibility=bool( + raw_issue.get("_von_source_archive") + ), + ) action = "updated" summary["updated"] += 1 else: @@ -2310,14 +2552,57 @@ def import_jira_issues_to_tasks( action = "would_create" summary["would_create"] += 1 else: + create_fields = {} + if raw_issue.get("_von_source_archive"): + create_fields = { + key: update_fields_payload[key] + for key in ( + "priority", + "start_date", + "due_date", + "components", + "fix_versions", + "sprint_values", + "backlog_rank", + "project_concept_id", + "collection_concept_ids", + ) + if key in update_fields_payload + } created = create_task( + **create_fields, title=title, description=description, assignee_concept_id=assignee_concept_id, - created_by_concept_id=creator_concept_id or actor_concept_id, + created_by_concept_id=actor_concept_id or creator_concept_id, organisation_concept_id=organisation_concept_id, task_source_id=JIRA_IMPORTED_TASK_SOURCE_ID, reference_code=issue_key, + visibility_owner_concept_id=( + actor_concept_id + if raw_issue.get("_von_source_archive") + else None + ), + agent_creation_fingerprint=( + f"jira_import:{actor_concept_id}:{organisation_concept_id}:{raw_issue.get('self')}:{raw_issue.get('id')}" + if raw_issue.get("_von_source_archive") + else None + ), + initial_external_references=( + { + "jira": { + "external_id": issue_key, + "issue_key": issue_key, + "issue_id": raw_issue.get("id"), + "issue_url": raw_issue.get("self"), + "project_key": project_key, + "source_archive": raw_issue["_von_source_archive"], + "import_phase": "initialising", + } + } + if raw_issue.get("_von_source_archive") + else None + ), ) created_task_id = created.get("task_concept_id") if not isinstance(created_task_id, str) or not created_task_id.strip(): @@ -2325,11 +2610,22 @@ def import_jira_issues_to_tasks( f"create_task did not return task_concept_id for {issue_key}" ) task_id = created_task_id.strip() - update_task_fields( - task_id, - fields=update_fields_payload, - actor_concept_id=actor_concept_id, - ) + if raw_issue.get("_von_source_archive"): + update_fields_payload = { + key: value + for key, value in update_fields_payload.items() + if _projection_value(key, created.get(key)) + != _projection_value(key, value) + } + if update_fields_payload: + update_task_fields( + task_id, + fields=update_fields_payload, + actor_concept_id=actor_concept_id, + preserve_visibility=bool( + raw_issue.get("_von_source_archive") + ), + ) action = "created" summary["created"] += 1 existing_cache[issue_key] = task_id @@ -2400,6 +2696,31 @@ def import_jira_issues_to_tasks( watcher_concept_ids=watcher_concept_ids, ) reference_payload["imported_activity"] = imported_activity + aliases = set(previous_reference.get("previous_issue_keys") or []) + old_key = previous_reference.get("issue_key") or previous_reference.get( + "external_id" + ) + if old_key and old_key != issue_key: + aliases.add(old_key) + reference_payload["previous_issue_keys"] = sorted(aliases) + if source_projection is not None: + reference_payload["last_import_projection"] = source_projection + reference_payload["sync_conflicts"] = sync_conflicts + previous_archives = list( + previous_reference.get("source_archive_history") or [] + ) + previous_archive = previous_reference.get("source_archive") + current_archive = reference_payload.get("source_archive") + if previous_archive and previous_archive.get("content_sha256") != ( + current_archive or {} + ).get("content_sha256"): + if not any( + item.get("content_sha256") + == previous_archive.get("content_sha256") + for item in previous_archives + ): + previous_archives.append(previous_archive) + reference_payload["source_archive_history"] = previous_archives upsert_task_external_reference( task_id, source_system=_JIRA_SOURCE_SYSTEM, @@ -2436,12 +2757,97 @@ def import_jira_issues_to_tasks( "participant_mappings": participant_resolution, "project_key": project_key, "project_name": project_name, + "sync_conflicts": sync_conflicts, + "guard_existing_hierarchy": bool( + raw_issue.get("_von_source_archive") + and existing_task + and previous_reference.get("import_phase") != "initialising" + ), + "hierarchy_baseline": { + "parent": previous_reference.get("parent_issue_key"), + "epic": previous_reference.get("epic_issue_key"), + }, } ) + relation_summary = repair_jira_task_relations( + issue_results, + actor_concept_id=actor_concept_id, + organisation_concept_id=organisation_concept_id, + dry_run=dry_run, + ) + summary["mapped_relations"] += relation_summary["mapped_relations"] + summary["dropped_relations"] += relation_summary["dropped_relations"] + + project_parity_report = _build_project_parity_report(project_observations) + pilot_validation = _build_pilot_validation_report( + project_parity=project_parity_report, + summary=summary, + ) + + return { + "success": True, + "source_system": _JIRA_SOURCE_SYSTEM, + "dry_run": bool(dry_run), + "update_existing": bool(update_existing), + "summary": summary, + "issues": issue_results, + "project_parity": project_parity_report, + "pilot_validation": pilot_validation, + } + + +def _check_imported_hierarchy_update( + row, kind, target_id, *, organisation_concept_id, cache, task_id_by_issue_key +): + if not row.get("guard_existing_hierarchy"): + return + current = get_task(row["task_concept_id"]).get(f"{kind}_task_concept_id") + if current == target_id: + return + previous_key = row.get("hierarchy_baseline", {}).get(kind) + previous_id = ( + ( + task_id_by_issue_key.get(previous_key) + or _resolve_existing_task_id( + jira_issue_key=previous_key, + organisation_concept_id=organisation_concept_id, + cache=cache, + ) + ) + if previous_key + else None + ) + if current != previous_id: + raise InvalidTaskDataError( + f"Native {kind} edit preserved; source hierarchy requires reconciliation" + ) + + +def repair_jira_task_relations( + issue_results, + *, + actor_concept_id, + organisation_concept_id=None, + dry_run=False, + only_unresolved=False, +): + """Repair cross-batch references from retained import rows without refetching Jira.""" + summary = {"mapped_relations": 0, "dropped_relations": 0} + existing_cache = {} + task_id_by_issue_key = { + row["jira_issue_key"]: row["task_concept_id"] + for row in issue_results + if row.get("task_concept_id") and row.get("jira_issue_key") + } # Pass 2: apply hierarchy and link relationships once task IDs are known. seen_links: set[tuple[str, str, str]] = set() for issue_row in issue_results: + if only_unresolved and not any( + row.get("status") == "dropped" + for row in issue_row.get("relation_results", []) + ): + continue issue_key = issue_row.get("jira_issue_key") if not isinstance(issue_key, str): continue @@ -2454,6 +2860,9 @@ def import_jira_issues_to_tasks( # Dry-run still reports relation intent, but does not mutate. pass + if issue_row.get("action") == "skipped_von_authoritative": + continue + issue_row["relation_results"] = [] relation_results_raw = issue_row.get("relation_results") if isinstance(relation_results_raw, list): relation_results_list: list[Dict[str, Any]] = relation_results_raw @@ -2471,6 +2880,14 @@ def import_jira_issues_to_tasks( if isinstance(target_task_id, str) and target_task_id: try: if not dry_run and not source_task_id.startswith("planned:"): + _check_imported_hierarchy_update( + issue_row, + "parent", + target_task_id, + organisation_concept_id=organisation_concept_id, + cache=existing_cache, + task_id_by_issue_key=task_id_by_issue_key, + ) set_task_parent( source_task_id, target_task_id, @@ -2516,6 +2933,14 @@ def import_jira_issues_to_tasks( if isinstance(target_task_id, str) and target_task_id: try: if not dry_run and not source_task_id.startswith("planned:"): + _check_imported_hierarchy_update( + issue_row, + "epic", + target_task_id, + organisation_concept_id=organisation_concept_id, + cache=existing_cache, + task_id_by_issue_key=task_id_by_issue_key, + ) set_task_epic( source_task_id, target_task_id, @@ -2635,19 +3060,4 @@ def import_jira_issues_to_tasks( ) summary["dropped_relations"] += 1 - project_parity_report = _build_project_parity_report(project_observations) - pilot_validation = _build_pilot_validation_report( - project_parity=project_parity_report, - summary=summary, - ) - - return { - "success": True, - "source_system": _JIRA_SOURCE_SYSTEM, - "dry_run": bool(dry_run), - "update_existing": bool(update_existing), - "summary": summary, - "issues": issue_results, - "project_parity": project_parity_report, - "pilot_validation": pilot_validation, - } + return summary diff --git a/src/backend/services/jira_task_migration_runner_service.py b/src/backend/services/jira_task_migration_runner_service.py index 491551535..64a866975 100644 --- a/src/backend/services/jira_task_migration_runner_service.py +++ b/src/backend/services/jira_task_migration_runner_service.py @@ -19,6 +19,7 @@ from dataclasses import dataclass, replace from datetime import datetime, timezone import json +import hashlib from pathlib import Path from typing import TYPE_CHECKING, Any, Iterator, Mapping, Sequence @@ -88,6 +89,8 @@ class JiraTaskMigrationOptions: report_path: Path | None = DEFAULT_JIRA_TASK_MIGRATION_REPORT_PATH updated_within_hours: int | None = None include_done: bool = False + preserve_source: bool = False + resume: bool = False min_issue_number: int | None = None max_issue_number: int | None = None @@ -105,7 +108,6 @@ def build_default_jira_task_migration_jql( ) clauses = [ f"project = {project_key_clean}", - "issuetype in (Task, Subtask, Epic)", ] if not include_done: clauses.append("statusCategory != Done") @@ -200,6 +202,8 @@ def normalise_jira_task_migration_options( include_done=bool(options.include_done), min_issue_number=min_issue_number, max_issue_number=max_issue_number, + preserve_source=bool(options.preserve_source), + resume=bool(options.resume), ) @@ -220,6 +224,7 @@ async def discover_jira_issue_docs( proxy = await get_jira_proxy() next_page_token: str | None = None issue_docs: list[dict[str, Any]] = [] + seen_tokens: set[str] = set() while True: payload = await proxy.search( @@ -232,11 +237,20 @@ async def discover_jira_issue_docs( next_page_token = ( payload.get("nextPageToken") if isinstance(payload, Mapping) else None ) - if not isinstance(raw_issues, list) or not raw_issues: - break + if not isinstance(raw_issues, list) or payload.get("success") is False: + raise RuntimeError( + "Jira discovery failed; an error is not an empty project" + ) + if not raw_issues and next_page_token: + raise RuntimeError("Jira discovery returned an empty non-terminal page") issue_docs.extend(item for item in raw_issues if isinstance(item, dict)) if not isinstance(next_page_token, str) or not next_page_token.strip(): + if payload.get("isLast") is False: + raise RuntimeError("Jira discovery omitted the next page token") break + if next_page_token in seen_tokens: + raise RuntimeError("Jira discovery pagination stalled") + seen_tokens.add(next_page_token) return issue_docs @@ -298,13 +312,17 @@ def _extract_missing_targets( def _build_gateway() -> InternalMCPGateway: from ..integrations.internal_mcp import build_default_catalogue - from ..integrations.internal_mcp.gateway import InternalMCPGateway + from ..integrations.internal_mcp.gateway import ( + InternalMCPGateway, + internal_mcp_actor_context_is_trusted_local_operator, + ) from ..integrations.internal_mcp.transport import InternalMCPTransport return InternalMCPGateway( catalogue=build_default_catalogue(), transport=InternalMCPTransport(), enabled=True, + trusted_actor_payload_fallback=internal_mcp_actor_context_is_trusted_local_operator(), ) @@ -317,12 +335,52 @@ def _run_import_batches( ) -> dict[str, Any]: reports: list[dict[str, Any]] = [] aggregate_summary: Counter[str] = Counter() - - for pass_index in range(options.passes): + checkpoint = ( + options.report_path.with_suffix(f".{run_label}.checkpoint.json") + if options.report_path is not None + else None + ) + signature_payload = { + "options": { + key: value + for key, value in _options_to_report_dict(options).items() + if key not in {"resume", "report_path"} + }, + "source_versions": [ + (item.get("id"), item.get("key"), item.get("fields", {}).get("updated")) + for item in issue_docs + ], + "run_label": run_label, + } + signature = hashlib.sha256( + json.dumps(signature_payload, sort_keys=True).encode() + ).hexdigest() + if options.resume and checkpoint and checkpoint.exists(): + saved = json.loads(checkpoint.read_text(encoding="utf-8")) + if saved.get("signature") != signature: + raise ValueError( + "Migration checkpoint scope/source changed; start a fresh run for the new delta" + ) + reports = saved.get("completed_batches", []) + for report in reports: + _merge_counter(aggregate_summary, report.get("summary")) + completed_count = len(reports) + batch_index = 0 + + # The historical extra pass only repaired cross-batch relationships. Full + # source capture is followed by that same repair using the retained rows. + capture_passes = 1 if options.preserve_source else options.passes + for pass_index in range(capture_passes): for batch in _iter_batches(issue_docs, options.batch_size): + batch_index += 1 + if batch_index <= completed_count: + continue payload = { - "issue_keys": [dict(item) for item in batch if isinstance(item, Mapping)], + "issue_keys": [ + dict(item) for item in batch if isinstance(item, Mapping) + ], "dry_run": options.dry_run, + "preserve_source": options.preserve_source, "update_existing": True, "include_watchers": False, "namespace": options.namespace, @@ -338,16 +396,51 @@ def _run_import_batches( ), } result = gateway.invoke("task_import_jira_issues", payload).payload - if not isinstance(result, dict) or result.get("success") is not True: + succeeded = isinstance(result, dict) and result.get("success") is True + if succeeded: + reports.append(result) + _merge_counter(aggregate_summary, result.get("summary")) + if checkpoint is not None: + write_jira_task_migration_report( + { + "signature": signature, + "run_label": run_label, + "pass_index": pass_index, + "completed_batches": reports, + "last_batch": result, + }, + report_path=checkpoint, + ) + if not succeeded: raise RuntimeError( f"{run_label} pass {pass_index + 1} failed: {result}" ) - reports.append(result) - _merge_counter(aggregate_summary, result.get("summary")) + relation_repair = None + if options.preserve_source and reports: + from .jira_task_import_service import repair_jira_task_relations + + rows = [row for report in reports for row in report.get("issues", [])] + relation_repair = repair_jira_task_relations( + rows, + actor_concept_id=options.actor_concept_id, + organisation_concept_id=options.organisation_concept_id, + dry_run=options.dry_run, + only_unresolved=True, + ) + relations = [ + relation for row in rows for relation in row.get("relation_results", []) + ] + aggregate_summary["mapped_relations"] = sum( + row.get("status") in {"mapped", "would_map"} for row in relations + ) + aggregate_summary["dropped_relations"] = sum( + row.get("status") == "dropped" for row in relations + ) return { "run_label": run_label, - "passes": options.passes, + "passes": capture_passes, + "relation_repair": relation_repair, "batch_size": options.batch_size, "input_issue_count": len(issue_docs), "batch_count": len(reports), @@ -407,6 +500,8 @@ def _options_to_report_dict( "report_path": str(options.report_path) if options.report_path else None, "updated_within_hours": options.updated_within_hours, "include_done": options.include_done, + "preserve_source": options.preserve_source, + "resume": options.resume, "min_issue_number": options.min_issue_number, "max_issue_number": options.max_issue_number, } @@ -418,7 +513,9 @@ def write_jira_task_migration_report( report_path: Path, ) -> Path: report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + temporary = report_path.with_suffix(report_path.suffix + ".tmp") + temporary.write_text(json.dumps(report, indent=2), encoding="utf-8") + temporary.replace(report_path) return report_path @@ -477,7 +574,11 @@ async def run_jira_task_migration( ] selection_mode = "explicit_jql" - if not options.include_done and options.updated_within_hours is None and not options.only_missing: + if ( + not options.include_done + and options.updated_within_hours is None + and not options.only_missing + ): selection_mode = "current_scope" if options.updated_within_hours is not None: selection_mode = "incremental_recent_window" @@ -511,9 +612,7 @@ async def run_jira_task_migration( options=options, run_label="current_scope", ) - report["current_scope"] = { - key: value for key, value in main_run.items() if key != "reports" - } + report["current_scope"] = {key: value for key, value in main_run.items()} referenced_target_keys = main_run.get("missing_target_issue_keys") or [] if options.import_referenced_targets and referenced_target_keys: diff --git a/src/backend/services/jira_task_reconciliation_service.py b/src/backend/services/jira_task_reconciliation_service.py index ad4eb4feb..b50adbbef 100644 --- a/src/backend/services/jira_task_reconciliation_service.py +++ b/src/backend/services/jira_task_reconciliation_service.py @@ -9,8 +9,12 @@ from __future__ import annotations import asyncio +import base64 +import hashlib from dataclasses import dataclass from typing import Any, Mapping +from collections import Counter, defaultdict +from datetime import datetime, timezone from ..utils.jira_issue_key_utils import ( extract_jira_issue_number, @@ -28,6 +32,291 @@ _MAX_GAP_BLOCK_SIZE = 500 +def reconcile_retained_project( + *, source_issues, project_concept_id, actor_concept_id, verify_archives=True +): + """Reconcile identities, versions, memberships and retained bytes explicitly. + + Reads canonical repositories and archive services. Missing source rows never + authorise deleting a task or archive; they remain inspectable discrepancies. + This is retention evidence, not a substitute for ordinary-use/restore proof. + """ + from ..db.repositories.concepts_repository import ConceptsRepository + from .task_project_service import get_task_project, get_task_collection + from .jira_source_retention_service import read_source_archive + from .task_management_service import ( + PREDICATE_HAS_PARENT_TASK, + PREDICATE_HAS_EPIC_TASK, + TASK_LINK_TYPE_TO_PREDICATE, + ) + from .jira_task_import_service import ( + _extract_parent_issue_key, + _extract_epic_issue_key, + _extract_issue_links, + ) + + project = get_task_project(project_concept_id) + project_key = project["key"] + source_by_key, source_ids = {}, Counter() + problems, verified, hashes = [], [], [] + for issue in source_issues: + key = issue.get("key") + if ( + not key + or issue.get("fields", {}).get("project", {}).get("key") != project_key + ): + problems.append({"kind": "source_scope_mismatch", "key": key}) + continue + if key in source_by_key: + problems.append({"kind": "duplicate_source_key", "key": key}) + source_by_key[key] = issue + source_ids[str(issue.get("id"))] += 1 + problems.extend( + {"kind": "duplicate_source_id", "id": key} + for key, count in source_ids.items() + if count != 1 + ) + docs = list( + ConceptsRepository.find( + { + "relationships.is_an_instance_of": "#V#task_specification", + "$or": [ + {"metadata.external_references.jira.project_key": project_key}, + {"metadata.project_concept_id": project_concept_id}, + ], + }, + projection={"concept_id": 1, "metadata": 1, "relationships": 1}, + ) + ) + by_key = defaultdict(list) + for doc in docs: + ref = doc.get("metadata", {}).get("external_references", {}).get("jira", {}) + if ref.get("external_id"): + by_key[ref["external_id"]].append(doc) + missing, extra = sorted(set(source_by_key) - set(by_key)), sorted( + set(by_key) - set(source_by_key) + ) + problems.extend({"kind": "missing_task", "key": key} for key in missing) + problems.extend({"kind": "source_absent_or_moved", "key": key} for key in extra) + collections = [ + get_task_collection(value) for value in project["collection_concept_ids"] + ] + required_collections = { + c["concept_id"] + for c in collections + if c["selection"]["kind"] == "project_membership" + } + for collection in collections: + if project_concept_id not in collection["project_concept_ids"]: + problems.append( + { + "kind": "collection_project_mismatch", + "collection": collection["concept_id"], + } + ) + source_relations = {} + target_keys = set() + for key, issue in source_by_key.items(): + fields = issue.get("fields", {}) + relations = [] + for kind, target in ( + ("parent", _extract_parent_issue_key(fields)), + ("epic", _extract_epic_issue_key(fields)), + ): + if target: + relations.append({"kind": kind, "target_issue_key": target}) + relations.extend( + {"kind": "link", **value} for value in _extract_issue_links(fields) + ) + source_relations[key] = relations + target_keys.update(row["target_issue_key"] for row in relations) + external_targets = target_keys - set(by_key) + target_by_key = dict(by_key) + if external_targets: + for doc in ConceptsRepository.find( + { + "relationships.is_an_instance_of": "#V#task_specification", + "metadata.external_references.jira.external_id": { + "$in": sorted(external_targets) + }, + }, + projection={ + "concept_id": 1, + "metadata.external_references.jira": 1, + "relationships": 1, + }, + ): + key = ( + doc.get("metadata", {}) + .get("external_references", {}) + .get("jira", {}) + .get("external_id") + ) + target_by_key.setdefault(key, []).append(doc) + for key, issue in source_by_key.items(): + candidates = by_key.get(key, []) + if len(candidates) != 1: + if candidates: + problems.append( + { + "kind": "duplicate_task", + "key": key, + "task_ids": [x["concept_id"] for x in candidates], + } + ) + continue + doc = candidates[0] + meta = doc.get("metadata", {}) + ref = meta.get("external_references", {}).get("jira", {}) + if str(ref.get("issue_id")) != str(issue.get("id")): + problems.append({"kind": "source_id_mismatch", "key": key}) + if ref.get("updated") != issue.get("fields", {}).get("updated"): + problems.append({"kind": "source_version_mismatch", "key": key}) + if meta.get( + "project_concept_id" + ) != project_concept_id or not required_collections.issubset( + meta.get("collection_concept_ids") or [] + ): + problems.append({"kind": "task_membership_mismatch", "key": key}) + if ref.get("sync_conflicts"): + problems.append( + { + "kind": "unresolved_native_edits", + "key": key, + "fields": ref["sync_conflicts"], + } + ) + archive_ref = ref.get("source_archive") + if not archive_ref or not archive_ref.get("complete"): + problems.append({"kind": "missing_or_incomplete_original", "key": key}) + continue + if verify_archives: + try: + archive = read_source_archive( + archive_ref, actor_concept_id=actor_concept_id + ) + if ( + not archive.get("complete") + or str(archive["source"].get("id")) != str(issue.get("id")) + or archive["source"]["fields"].get("updated") + != issue["fields"].get("updated") + ): + raise ValueError( + "Retained source identity/version/completeness mismatch" + ) + for item in archive.get("binaries", []): + data = base64.b64decode( + item["source"]["content_base64"], validate=True + ) + if ( + not item.get("complete") + or len(data) != item["size_bytes"] + or hashlib.sha256(data).hexdigest() != item["sha256"] + ): + raise ValueError( + "Retained attachment bytes do not match their manifest" + ) + verified.append(key) + hashes.extend( + { + "issue_key": key, + "attachment_id": item["id"], + "sha256": item["sha256"], + "size_bytes": item["size_bytes"], + } + for item in archive.get("binaries", []) + ) + except Exception as exc: + problems.append( + {"kind": "archive_readback_failed", "key": key, "error": str(exc)} + ) + for relation in source_relations[key]: + target_key = relation["target_issue_key"] + targets = target_by_key.get(target_key, []) + if len(targets) != 1: + problems.append( + { + "kind": "unresolved_relation_target", + "key": key, + "target_key": target_key, + "candidate_count": len(targets), + } + ) + continue + kind = relation["kind"] + if kind == "parent": + predicates = [PREDICATE_HAS_PARENT_TASK, PREDICATE_HAS_EPIC_TASK] + elif kind == "epic": + predicates = [PREDICATE_HAS_EPIC_TASK] + else: + predicate = TASK_LINK_TYPE_TO_PREDICATE.get(relation.get("link_type")) + if not predicate: + problems.append( + { + "kind": "unmapped_source_link", + "key": key, + "relation": relation, + } + ) + continue + predicates = [predicate] + represented = { + value + for predicate in predicates + for value in doc.get("relationships", {}).get(predicate, []) + } + if targets[0]["concept_id"] not in represented: + problems.append( + {"kind": "relation_mismatch", "key": key, "relation": relation} + ) + project_archive = project.get("source_archive") + if not project_archive or not project_archive.get("complete"): + problems.append({"kind": "project_source_incomplete"}) + elif verify_archives: + try: + archived_project = read_source_archive( + project_archive, actor_concept_id=actor_concept_id + ) + if ( + not archived_project.get("complete") + or str(archived_project["source"].get("id")) != project["source_id"] + ): + raise ValueError("Project archive identity or completeness mismatch") + except Exception as exc: + problems.append( + {"kind": "project_archive_readback_failed", "error": str(exc)} + ) + return { + "schema": "jira_project_retention_reconciliation.v1", + "generated_at": datetime.now(timezone.utc).isoformat(), + "project_concept_id": project_concept_id, + "project_key": project_key, + "source_issue_count": len(source_by_key), + "represented_issue_count": len(set(source_by_key) & set(by_key)), + "missing_issue_count": len(missing), + "missing_issue_keys": missing, + "missing_type_counts": dict( + Counter( + (source_by_key[key].get("fields", {}).get("issuetype") or {}).get( + "name", "Unknown" + ) + for key in missing + ) + ), + "source_type_counts": dict( + Counter( + (issue.get("fields", {}).get("issuetype") or {}).get("name", "Unknown") + for issue in source_by_key.values() + ) + ), + "verified_original_count": len(verified), + "binary_hashes": hashes, + "problems": problems, + "retention_reconciled": verify_archives and not problems, + "cutover_ready": False, + } + + @dataclass(frozen=True) class JiraTaskGapScanOptions: project_key: str = DEFAULT_JIRA_TASK_MIGRATION_PROJECT_KEY diff --git a/src/backend/services/task_management_service.py b/src/backend/services/task_management_service.py index 99781d128..f4a564298 100644 --- a/src/backend/services/task_management_service.py +++ b/src/backend/services/task_management_service.py @@ -1014,7 +1014,18 @@ def _append_task_history_event( if touch_updated_at: update_doc.setdefault("$set", {}) update_doc["$set"]["updated_at"] = _now() - ConceptsRepository.update_one({"concept_id": task_concept_id}, update_doc) + query: Dict[str, Any] = {"concept_id": task_concept_id} + signature = (details or {}).get("import_signature") + if signature: + query[f"metadata.{TASK_METADATA_KEY_HISTORY}"] = { + "$not": { + "$elemMatch": { + "event_type": event_type, + "details.import_signature": signature, + } + } + } + ConceptsRepository.update_one(query, update_doc) return event @@ -1023,14 +1034,36 @@ def _append_task_metadata_entry( task_concept_id: str, metadata_key: str, entry: Dict[str, Any], -) -> None: - ConceptsRepository.update_one( - {"concept_id": task_concept_id}, +) -> Dict[str, Any]: + query: Dict[str, Any] = {"concept_id": task_concept_id} + source = entry.get("source") or {} + source_match = None + if source.get("source_system") and source.get("external_id"): + source_match = { + "source.source_system": source["source_system"], + "source.external_id": source["external_id"], + } + query[f"metadata.{metadata_key}"] = {"$not": {"$elemMatch": source_match}} + result = ConceptsRepository.update_one( + query, { "$push": {f"metadata.{metadata_key}": entry}, "$set": {"updated_at": _now()}, }, ) + if source_match and getattr(result, "matched_count", 1) == 0: + _, doc = _get_task_doc(task_concept_id) + for existing in _list_metadata_items(doc, metadata_key): + previous_source = existing.get("source") or {} + if all( + previous_source.get(key.split(".")[-1]) == value + for key, value in source_match.items() + ): + return existing + raise TaskManagementError( + "Imported activity was not stored or found on read-back" + ) + return entry def _replace_task_metadata_list( @@ -1268,6 +1301,10 @@ def create_task( reference_code: str | None = None, agent_creation_fingerprint: str | None = None, agent_creation_request_id: str | None = None, + project_concept_id: str | None = None, + collection_concept_ids: list[str] | None = None, + visibility_owner_concept_id: str | None = None, + initial_external_references: Mapping[str, Any] | None = None, ) -> Dict[str, Any]: """Create a new task as a Vontology concept. @@ -1413,6 +1450,15 @@ def create_task( now = _now() + if project_concept_id or collection_concept_ids: + from .task_project_service import validate_task_membership + + project, collection_concept_ids = validate_task_membership( + project_concept_id, collection_concept_ids + ) + if project and organisation_concept_id is None: + organisation_concept_id = project.get("organisation_concept_id") + # Build relationships relationships: Dict[str, Any] = { "is_an_instance_of": [TASK_SPECIFICATION_TYPE_ID, *canonical_task_type_ids], @@ -1437,6 +1483,10 @@ def create_task( visible_to_users.append(assignee_concept_id) if report_to_concept_id and report_to_concept_id not in visible_to_users: visible_to_users.append(report_to_concept_id) + if visibility_owner_concept_id: + # Import provenance and assignment do not confer access to a private + # source corpus. Used by the trusted importer, not exposed as a UI field. + visible_to_users = [visibility_owner_concept_id] if visible_to_users: relationships = set_specific_to_user_values(relationships, visible_to_users) @@ -1481,10 +1531,14 @@ def create_task( TASK_METADATA_KEY_ATTACHMENTS: [], TASK_METADATA_KEY_WORKLOG: [], TASK_METADATA_KEY_HISTORY: [], - TASK_METADATA_KEY_EXTERNAL_REFERENCES: {}, + TASK_METADATA_KEY_EXTERNAL_REFERENCES: dict( + initial_external_references or {} + ), TASK_METADATA_KEY_BULK_TASK_COLLECTIONS: [], TASK_METADATA_KEY_CREATION_FINGERPRINT: agent_creation_fingerprint, TASK_METADATA_KEY_CREATION_REQUEST_ID: agent_creation_request_id, + "project_concept_id": project_concept_id, + "collection_concept_ids": collection_concept_ids or [], }, } @@ -1561,6 +1615,8 @@ def create_task( result = { "task_concept_id": task_concept_id, "title": title, + "project_concept_id": project_concept_id, + "collection_concept_ids": collection_concept_ids or [], "description": description, "status": TASK_STATUS_PENDING, "priority": priority, @@ -2117,6 +2173,8 @@ def _build_task_response( "worklog_total_minutes": worklog_total_minutes, "history_count": len(history), "external_references": external_references, + "project_concept_id": metadata.get("project_concept_id"), + "collection_concept_ids": metadata.get("collection_concept_ids", []), "bulk_task_collections": bulk_task_collections, "hidden_by_default_bulk_task_collections": ( hidden_by_default_bulk_task_collections @@ -2285,7 +2343,9 @@ def update_task_status( return updated_task -def assign_task(task_concept_id: str, assignee_concept_id: str) -> Dict[str, Any]: +def assign_task( + task_concept_id: str, assignee_concept_id: str, *, update_visibility: bool = True +) -> Dict[str, Any]: """Assign or reassign a task to a person or organisation. Args: @@ -2331,14 +2391,16 @@ def assign_task(task_concept_id: str, assignee_concept_id: str) -> Dict[str, Any action="add", ) - # Also add new assignee to the canonical user visibility predicate. - for predicate in SPECIFIC_TO_USER_PREDICATES_WRITE: - ConceptsRepository.mutate_relationship_edge( - source_id=task_concept_id, - kind=predicate, - target_id=assignee_concept_id, - action="add", - ) + # Source assignment records do not grant the source participant new access. + if update_visibility: + # Also add new assignee to the canonical user visibility predicate. + for predicate in SPECIFIC_TO_USER_PREDICATES_WRITE: + ConceptsRepository.mutate_relationship_edge( + source_id=task_concept_id, + kind=predicate, + target_id=assignee_concept_id, + action="add", + ) # Update timestamp now = _now() @@ -3299,6 +3361,8 @@ def _actor_scoped_task_query_candidates( def search_tasks( *, query: str | None = None, + project_concept_id: str | None = None, + collection_concept_id: str | None = None, status_filter: str | None = None, statuses: list[str] | None = None, task_type_ids: list[str] | str | None = None, @@ -3347,6 +3411,14 @@ def search_tasks( query_filter: Dict[str, Any] = { "relationships.is_an_instance_of": TASK_SPECIFICATION_TYPE_ID } + if project_concept_id: + query_filter["metadata.project_concept_id"] = project_concept_id + if collection_concept_id: + from .task_project_service import collection_task_query + + query_filter.setdefault("$and", []).append( + collection_task_query(collection_concept_id) + ) if organisation_concept_id: org_id = _normalise_optional_concept_id(organisation_concept_id) if not org_id: @@ -3977,7 +4049,11 @@ def create_subtask( organisation_concept_id: str | None = None, originating_session_id: str | None = None, ) -> Dict[str, Any]: - parent_task_concept_id, _ = _get_task_doc(parent_task_concept_id) + parent_task_concept_id, parent_doc = _get_task_doc(parent_task_concept_id) + parent_metadata = parent_doc.get("metadata") or {} + project_id = parent_metadata.get("project_concept_id") + if project_id and organisation_concept_id is None: + organisation_concept_id = parent_metadata.get(TASK_METADATA_KEY_ORGANISATION) created = create_task( title=title, description=description, @@ -3989,6 +4065,8 @@ def create_subtask( priority=priority, organisation_concept_id=organisation_concept_id, originating_session_id=originating_session_id, + project_concept_id=project_id, + collection_concept_ids=parent_metadata.get("collection_concept_ids") or [], ) subtask_id = str(created.get("task_concept_id")) subtask = set_task_parent( @@ -4223,11 +4301,13 @@ def add_task_comment( } if isinstance(source, Mapping): comment["source"] = dict(source) - _append_task_metadata_entry( + stored_entry = _append_task_metadata_entry( task_concept_id=task_concept_id, metadata_key=TASK_METADATA_KEY_COMMENTS, entry=comment, ) + if stored_entry != comment: + return stored_entry try: _append_task_history_event( task_concept_id=task_concept_id, @@ -4311,6 +4391,79 @@ def find_task_comment_by_effect_fingerprint( return None +def add_task_attachment_bytes( + task_concept_id: str, + *, + data: bytes, + filename: str, + actor_concept_id: str, + media_type: str | None = None, + note: str | None = None, +) -> Dict[str, Any]: + """Store an attachment in Von and link its verified file-copy receipt.""" + from urllib.parse import quote + from .computer_file_copy_service import ( + import_bytes_file_copy, + fetch_file_copy_bytes, + ) + + task_concept_id, doc = _get_task_doc(task_concept_id) + if not actor_concept_id: + raise InvalidTaskDataError("An authenticated actor is required") + if not isinstance(data, bytes) or not data or len(data) > 100 * 1024 * 1024: + raise InvalidTaskDataError("Attachment must contain 1 byte to 100 MiB") + if not isinstance(filename, str) or not filename.strip(): + raise InvalidTaskDataError("filename is required") + org_id = doc.get("metadata", {}).get(TASK_METADATA_KEY_ORGANISATION) + digest = hashlib.sha256(data).hexdigest() + scope = hashlib.sha256( + f"{actor_concept_id}:{task_concept_id}".encode() + ).hexdigest()[:24] + receipt = import_bytes_file_copy( + data=data, + user_concept_id=actor_concept_id, + organisation_concept_id=org_id, + original_filename=filename.strip(), + content_type=media_type, + source_system="von_task_attachment", + source_identifier=task_concept_id, + blob_key=f"task-attachments/{scope}/{digest}", + metadata_in_attributes=True, + infer_typing=False, + maintain_relationship_inverses=False, + visibility_scope_mode="user_org_default" if org_id else "user_only_default", + ) + if not receipt.get("success"): + raise TaskManagementError("Attachment file was not stored") + file_id = receipt["concept_id"] + readback = fetch_file_copy_bytes( + file_copy_concept_id=file_id, + user_concept_id=actor_concept_id, + organisation_concept_id=org_id, + allow_large=True, + ) + if ( + not readback.get("success") + or hashlib.sha256(readback["data"]).hexdigest() != digest + ): + raise TaskManagementError("Attachment file read-back failed") + return add_task_attachment( + task_concept_id, + filename=filename, + uri=f"/von/api/files/{quote(file_id, safe='')}/download", + media_type=media_type, + size_bytes=len(data), + added_by_concept_id=actor_concept_id, + note=note, + file_copy_concept_id=file_id, + source={ + "source_system": "von_task_attachment", + "external_id": digest, + "sha256": digest, + }, + ) + + def add_task_attachment( task_concept_id: str, *, @@ -4350,11 +4503,13 @@ def add_task_attachment( ) if isinstance(normalised_file_copy_concept_id, str): attachment["file_copy_concept_id"] = normalised_file_copy_concept_id - _append_task_metadata_entry( + stored_entry = _append_task_metadata_entry( task_concept_id=task_concept_id, metadata_key=TASK_METADATA_KEY_ATTACHMENTS, entry=attachment, ) + if stored_entry != attachment: + return stored_entry try: _append_task_history_event( task_concept_id=task_concept_id, @@ -4429,11 +4584,13 @@ def add_task_worklog( } if isinstance(source, Mapping): entry["source"] = dict(source) - _append_task_metadata_entry( + stored_entry = _append_task_metadata_entry( task_concept_id=task_concept_id, metadata_key=TASK_METADATA_KEY_WORKLOG, entry=entry, ) + if stored_entry != entry: + return stored_entry try: _append_task_history_event( task_concept_id=task_concept_id, @@ -4539,6 +4696,7 @@ def update_task_fields( *, fields: Dict[str, Any], actor_concept_id: str | None = None, + preserve_visibility: bool = False, ) -> Dict[str, Any]: task_concept_id = _normalise_task_concept_id(task_concept_id) _, task_doc = _get_task_doc(task_concept_id) @@ -4547,6 +4705,30 @@ def update_task_fields( raise InvalidTaskDataError("fields must be a non-empty dict") existing_task = _build_task_response(task_doc) + membership = None + if "project_concept_id" in fields or "collection_concept_ids" in fields: + from .task_project_service import validate_task_membership + + project_id = fields.get( + "project_concept_id", existing_task.get("project_concept_id") + ) + collection_ids = fields.get( + "collection_concept_ids", existing_task.get("collection_concept_ids", []) + ) + if not isinstance(collection_ids, list) or any( + not isinstance(value, str) for value in collection_ids + ): + raise InvalidTaskDataError( + "collection_concept_ids must be a list of concept IDs" + ) + try: + _, collection_ids = validate_task_membership(project_id, collection_ids) + except ValueError as exc: + raise InvalidTaskDataError(str(exc)) from exc + membership = { + "project_concept_id": project_id, + "collection_concept_ids": collection_ids, + } # An explicit reference is validated before any other requested edits. product_field_present = "current_work_product_concept_id" in fields product_id = None @@ -4664,7 +4846,10 @@ def update_task_fields( unassign_task(task_concept_id) changed_fields.append("assignee_concept_id") else: - assign_task(task_concept_id, str(raw_assignee)) + if preserve_visibility: + assign_task(task_concept_id, str(raw_assignee), update_visibility=False) + else: + assign_task(task_concept_id, str(raw_assignee)) changed_fields.append("assignee_concept_id") created_by_field_present = ( @@ -5150,11 +5335,20 @@ def update_task_fields( "parent_task_concept_id", "epic_task_concept_id", "bulk_task_collections", + "project_concept_id", + "collection_concept_ids", } ) if unknown_keys: warnings.append(f"Ignored unsupported fields: {unknown_keys}") + if membership is not None: + ConceptsRepository.update_one( + {"concept_id": task_concept_id}, + {"$set": {f"metadata.{key}": value for key, value in membership.items()}}, + ) + changed_fields.extend(membership) + if changed_fields: ConceptsRepository.update_one( {"concept_id": task_concept_id}, @@ -5208,7 +5402,14 @@ def find_task_by_external_reference( query_base: Dict[str, Any] = { "relationships.is_an_instance_of": TASK_SPECIFICATION_TYPE_ID, - f"metadata.{TASK_METADATA_KEY_EXTERNAL_REFERENCES}.{source_key}.external_id": external_value, + "$or": [ + { + f"metadata.{TASK_METADATA_KEY_EXTERNAL_REFERENCES}.{source_key}.external_id": external_value + }, + { + f"metadata.{TASK_METADATA_KEY_EXTERNAL_REFERENCES}.{source_key}.previous_issue_keys": external_value + }, + ], } doc: Dict[str, Any] | None = None if organisation_concept_id: diff --git a/src/backend/services/task_project_service.py b/src/backend/services/task_project_service.py new file mode 100644 index 000000000..4b276d8df --- /dev/null +++ b/src/backend/services/task_project_service.py @@ -0,0 +1,365 @@ +"""Project and collection references for the canonical task store. + +These are represented identities with small inspectable reference records, not +a parallel task store. Membership never changes a task's audience. Source +configuration stays in file-copy archives rather than being copied to tasks. +""" + +from __future__ import annotations + +import hashlib +from datetime import UTC +from typing import Any +from urllib.parse import urlsplit + +from ..db.repositories.concepts_repository import ConceptsRepository +from ..security.access_control import can_access_concept +from .concept_service import ( + ConceptNotFoundError, + create_concept, + get_concept_by_concept_id_exact, + update_concept, +) + +PROJECT_RECORD = "task_project" +COLLECTION_RECORD = "task_collection" + + +def _find_concept(concept_id): + try: + return get_concept_by_concept_id_exact(concept_id) + except ConceptNotFoundError: + return None + + +def _record(concept_id, key): + if not can_access_concept(concept_id): + raise ValueError("Project or collection is not accessible") + doc = _find_concept(concept_id) + record = (doc.get("attributes") or {}).get(key) if doc else None + if not isinstance(record, dict): + raise ValueError("Project or collection not found") # noqa: TRY004 + return {"concept_id": concept_id, **record} + + +def get_task_project(project_concept_id): + return _record(project_concept_id, PROJECT_RECORD) + + +def get_task_collection(collection_concept_id): + return _record(collection_concept_id, COLLECTION_RECORD) + + +def list_project_collections(project): + """A shared project can reference views with a narrower audience.""" + return [ + get_task_collection(value) + for value in project["collection_concept_ids"] + if can_access_concept(value) + ] + + +def list_task_projects(*, limit=100, offset=0): + query = {f"attributes.{PROJECT_RECORD}": {"$exists": True}} + docs = ConceptsRepository.find( + query, + projection={"concept_id": 1, "attributes.task_project": 1}, + limit=max(1, min(int(limit), 200)), + skip=max(0, int(offset)), + ) + return [ + {"concept_id": doc["concept_id"], **doc["attributes"][PROJECT_RECORD]} + for doc in docs + ] + + +def ensure_jira_project( + project, + *, + actor_concept_id, + archive_reference=None, + organisation_concept_id=None, + existing_project_concept_id=None, +): + """Reuse a source-identity binding; deterministic IDs recover interrupted creates. + + A caller may associate a previously represented domain project explicitly. + Its broader publication context is never used for private source metadata. + """ + if not actor_concept_id or not project.get("id") or not project.get("key"): + raise ValueError("Actor and Jira project identity are required") + site = urlsplit(project.get("self", "")).netloc.lower() + if not site: + raise ValueError("Jira project site identity is required") + identity = hashlib.sha256( + f"{site}\n{project['id']}\n{actor_concept_id}\n{organisation_concept_id}".encode() + ).hexdigest()[:24] + project_id = f"#V#jira_project_{identity}" + collection_id = f"#V#jira_project_tasks_{identity}" + prior = _find_concept(project_id) + old = ((prior or {}).get("attributes") or {}).get(PROJECT_RECORD, {}) + if existing_project_concept_id and not can_access_concept( + existing_project_concept_id + ): + raise ValueError("Associated domain project is not accessible") + record: dict[str, Any] = { + **old, + "name": project.get("name", project["key"]), + "key": project["key"], + "source_system": "jira", + "source_site": site, + "source_id": str(project["id"]), + "organisation_concept_id": organisation_concept_id, + "source_url": project.get("self"), + "description": project.get("description", ""), + "collection_concept_ids": sorted( + set(old.get("collection_concept_ids", [])) | {collection_id} + ), + "writer": old.get("writer", "jira"), + "cutover_state": old.get("cutover_state", "importing"), + } + if archive_reference: + previous = old.get("source_archive") + if previous and previous.get("content_sha256") != archive_reference.get( + "content_sha256" + ): + history = list(old.get("source_archive_history") or []) + if not any( + item.get("content_sha256") == previous.get("content_sha256") + for item in history + ): + history.append(previous) + record["source_archive_history"] = history + record["source_archive"] = archive_reference + if existing_project_concept_id: + record["domain_project_concept_id"] = existing_project_concept_id + if prior: + update_concept(project_id, {f"attributes.{PROJECT_RECORD}": record}) + else: + create_concept( + name=record["name"], + concept_id=project_id, + parent_concept_ids=["#V#project"], + create_as_instance=True, + attributes={PROJECT_RECORD: record}, + created_by_concept_id=actor_concept_id, + organisation_concept_id=organisation_concept_id, + maintain_relationship_inverses=False, + visibility_scope_mode=( + "user_only_default" + if not organisation_concept_id + else "user_org_default" + ), + ) + collection = _find_concept(collection_id) + if not collection: + create_concept( + name=f"{record['name']} tasks", + concept_id=collection_id, + parent_concept_ids=["#V#first_order_collection"], + create_as_instance=True, + attributes={ + COLLECTION_RECORD: { + "name": f"{record['name']} tasks", + "project_concept_ids": [project_id], + "selection": { + "kind": "project_membership", + "project_concept_id": project_id, + }, + "source_project_key": project["key"], + "hidden_by_default": False, + } + }, + created_by_concept_id=actor_concept_id, + organisation_concept_id=organisation_concept_id, + maintain_relationship_inverses=False, + visibility_scope_mode=( + "user_only_default" + if not organisation_concept_id + else "user_org_default" + ), + ) + # Read both ends before publishing a usable context to the importer. + saved_project, saved_collection = get_task_project(project_id), get_task_collection( + collection_id + ) + if project_id not in saved_collection["project_concept_ids"]: + raise ValueError("Project collection association mismatch") + return { + "project_concept_id": project_id, + "collection_concept_ids": [collection_id], + "project": saved_project, + "collection": saved_collection, + } + + +def validate_task_membership(project_concept_id, collection_concept_ids): + project = get_task_project(project_concept_id) if project_concept_id else None + collections = list(dict.fromkeys(collection_concept_ids or [])) + for collection_id in collections: + collection = get_task_collection(collection_id) + if project and project_concept_id not in collection["project_concept_ids"]: + raise ValueError("Collection does not cover the selected project") + return project, collections + + +def find_jira_task_project( + *, project_key, source_site=None, organisation_concept_id=None +): + """Resolve only an accessible source binding, including for legacy sync jobs.""" + query = { + f"attributes.{PROJECT_RECORD}.key": project_key, + f"attributes.{PROJECT_RECORD}.source_system": "jira", + } + if source_site: + query[f"attributes.{PROJECT_RECORD}.source_site"] = source_site + if organisation_concept_id: + query[f"attributes.{PROJECT_RECORD}.organisation_concept_id"] = ( + organisation_concept_id + ) + docs = list( + ConceptsRepository.find( + query, + projection={"concept_id": 1, f"attributes.{PROJECT_RECORD}": 1}, + limit=2, + ) + ) + if len(docs) > 1: + raise ValueError("Ambiguous Jira project binding") + return ( + {"concept_id": docs[0]["concept_id"], **docs[0]["attributes"][PROJECT_RECORD]} + if docs + else None + ) + + +def associate_jira_source_collection( + *, + name, + source_id, + source_kind, + project_concept_ids, + issue_keys, + archive_reference, + actor_concept_id, +): + """Keep source board/filter membership as a snapshot, with native additions. + + The source query/configuration stays in its archive. It is never executed as + policy in Von. Explicit membership is refreshed only while Jira is writer. + """ + projects = [get_task_project(value) for value in dict.fromkeys(project_concept_ids)] + if not projects: + raise ValueError("A source collection must cover at least one project") + site = projects[0]["source_site"] + if any(p["source_site"] != site for p in projects): + raise ValueError("Source collection projects must belong to one Jira site") + identity = hashlib.sha256( + f"{actor_concept_id}:{site}:{source_kind}:{source_id}".encode() + ).hexdigest()[:24] + collection_id = f"#V#jira_task_collection_{identity}" + previous = _find_concept(collection_id) + old = (previous or {}).get("attributes", {}).get(COLLECTION_RECORD, {}) + record = { + **old, + "name": name, + "source_id": str(source_id), + "source_kind": source_kind, + "project_concept_ids": sorted(p["concept_id"] for p in projects), + "selection": { + "kind": "source_issue_keys", + "issue_keys": sorted(set(issue_keys)), + }, + "source_archive": archive_reference, + "hidden_by_default": False, + } + if previous and any(p.get("writer") == "von" for p in projects): + return get_task_collection(collection_id) + if previous: + update_concept(collection_id, {f"attributes.{COLLECTION_RECORD}": record}) + else: + create_concept( + name=name, + concept_id=collection_id, + parent_concept_ids=["#V#first_order_collection"], + create_as_instance=True, + attributes={COLLECTION_RECORD: record}, + created_by_concept_id=actor_concept_id, + maintain_relationship_inverses=False, + visibility_scope_mode="user_only_default", + ) + for project in projects: + ids = sorted(set(project["collection_concept_ids"]) | {collection_id}) + update_concept( + project["concept_id"], + {f"attributes.{PROJECT_RECORD}.collection_concept_ids": ids}, + ) + return get_task_collection(collection_id) + + +def collection_task_query(collection_concept_id): + collection = get_task_collection(collection_concept_id) + selection = collection.get("selection", {}) + clauses = [{"metadata.collection_concept_ids": collection_concept_id}] + if selection.get("kind") == "source_issue_keys": + clauses.append( + { + "metadata.external_references.jira.external_id": { + "$in": selection.get("issue_keys", []) + } + } + ) + elif selection.get("kind") == "project_membership": + clauses.append({"metadata.project_concept_id": selection["project_concept_id"]}) + return {"$or": clauses} + + +def set_task_project_writer(project_concept_id, *, writer, actor_concept_id, evidence): + """Record a bounded cutover or reversible pause with its reviewable receipts.""" + from datetime import datetime + + project = get_task_project(project_concept_id) + if writer not in {"jira", "von"} or not actor_concept_id: + raise ValueError("Writer and actor are required") + if writer == "von": + reconciliation = evidence.get("reconciliation", {}) + if reconciliation.get( + "project_concept_id" + ) != project_concept_id or not reconciliation.get("retention_reconciled"): + raise ValueError("A successful project content reconciliation is required") + if any( + not evidence.get(key) + for key in ( + "site_retention_receipt", + "ordinary_use_receipt", + "restore_receipt", + "final_delta_receipt", + ) + ): + raise ValueError( + "Site retention, ordinary-use, restore and final-delta receipts are required" + ) + history = list(project.get("writer_history") or []) + history.append( + { + "from": project.get("writer", "jira"), + "to": writer, + "at": datetime.now(UTC).isoformat(), + "actor_concept_id": actor_concept_id, + "evidence": evidence, + } + ) + update_concept( + project_concept_id, + { + f"attributes.{PROJECT_RECORD}.writer": writer, + f"attributes.{PROJECT_RECORD}.cutover_state": ( + "von_authoritative" if writer == "von" else "importing" + ), + f"attributes.{PROJECT_RECORD}.writer_history": history, + }, + ) + saved = get_task_project(project_concept_id) + if saved["writer"] != writer: + raise ValueError("Project writer read-back failed") + return saved diff --git a/src/backend/services/tool_metadata_service.py b/src/backend/services/tool_metadata_service.py index 3acdcbd23..74c3cda51 100644 --- a/src/backend/services/tool_metadata_service.py +++ b/src/backend/services/tool_metadata_service.py @@ -1227,6 +1227,26 @@ class ToolRequiredObligationMetadata: } _DEFAULT_VONTOLOGY_STDIO_EXPOSED_TOOL_NAMES = { + "task_get", + "task_search", + "task_update_fields", + "task_get_transitions", + "task_transition", + "task_create_subtask", + "task_set_parent", + "task_link", + "task_unlink", + "task_add_comment", + "task_list_comments", + "task_add_attachment", + "task_list_attachments", + "task_add_worklog", + "task_list_worklog", + "task_get_history", + "task_list_projects", + "task_get_project", + "task_get_collection", + "task_get_source_archive", "otter_collect_now", "otter_collection_status", "add_names", diff --git a/src/frontend/web/von_interface/static/js/components/taskPanel.js b/src/frontend/web/von_interface/static/js/components/taskPanel.js index ea9e69616..65df97a9d 100644 --- a/src/frontend/web/von_interface/static/js/components/taskPanel.js +++ b/src/frontend/web/von_interface/static/js/components/taskPanel.js @@ -5,7 +5,7 @@ * Tasks are stored as Vontology concepts and accessed via REST API. */ -import { deleteJson, getJson, patchJson, postJson, getUserContext } from '../apiService.js'; +import { deleteJson, getJson, patchJson, postJson, getUserContext, ensureUniqueWindowSessionId, WINDOW_SESSION_HEADER } from '../apiService.js'; import { activateTab } from '../tabNavigation.js'; import { showToast } from '../utils/toast.js'; import { renderMarkdownViaServer } from '../markdownUtils.js'; @@ -31,6 +31,10 @@ let _taskDetailState = {}; // taskId -> detail panel state let _selectedTaskId = ''; let _bulkTaskVisibility = 'exclude'; let _bulkTaskCollectionId = ''; +let _taskProjects = []; +let _selectedProjectId = ''; +let _selectedCollectionId = ''; +let _projectCollections = []; let _hiddenBulkTaskTotal = 0; let _hiddenBulkTaskCollections = []; let _lastTaskLoadTelemetry = null; @@ -176,6 +180,10 @@ function resetTaskPanelScopedState(activeOrganisationConceptId, { actorChanged = _globalTaskLoadGeneration += 1; _isLoading = false; _tasks = []; + _taskProjects = []; + _selectedProjectId = ""; + _selectedCollectionId = ""; + _projectCollections = []; _taskDetailState = {}; _selectedTaskId = ''; _globalTaskNextOffset = 0; @@ -194,6 +202,18 @@ function resetTaskPanelScopedState(activeOrganisationConceptId, { actorChanged = loadTaskGroupSelectionFromStorage(); } +function invalidateTaskSelectionPage() { + _globalTaskLoadGeneration += 1; + _isLoading = false; + _tasks = []; + _taskDetailState = {}; + _selectedTaskId = ''; + _globalTaskNextOffset = 0; + _globalTaskHasMore = false; + _globalTaskTotal = null; + return _globalTaskLoadGeneration; +} + function refreshTaskPanelAfterScopeChange({ allowLoads = true } = {}) { renderTaskList(); renderTaskQueueActivity(); @@ -203,7 +223,13 @@ function refreshTaskPanelAfterScopeChange({ allowLoads = true } = {}) { if (!allowLoads) return; if (_isGlobalTabMode) { - void Promise.all([loadGlobalTasks(), loadTaskQueueActivity()]); + const generation = _globalTaskLoadGeneration; + void (async () => { + await ensureTaskTaxonomyLoaded(); + if (generation !== _globalTaskLoadGeneration) return; + renderGlobalTasksTabContent(); + await Promise.all([loadGlobalTasks(), loadTaskQueueActivity()]); + })(); } else if (_isVisible) { void refreshTasks(); } @@ -317,6 +343,8 @@ function getTaskSourceOptions() { } function getTaskPrimaryType(task) { + const sourceType = task?.external_references?.jira?.issue_type; + if (sourceType) return { label: sourceType, concept_id: null }; if (task?.primary_task_type_id) { return { concept_id: task.primary_task_type_id, @@ -494,7 +522,12 @@ function finishTaskLoadTelemetry(telemetry, status, label, metadata = {}) { function startTaskLoadProgressTicker(telemetry) { if (typeof setInterval !== 'function') return () => {}; + const generation = _globalTaskLoadGeneration; const intervalId = setInterval(() => { + if (generation !== _globalTaskLoadGeneration) { + clearInterval(intervalId); + return; + } if (!telemetry || telemetry.status !== 'loading') return; updateTaskLoadElapsed(telemetry); renderTaskLoadProgress(telemetry.current_label, telemetry); @@ -588,6 +621,12 @@ function buildGlobalTasksUrl({ offset = 0, includeBulkSummary = true } = {}) { } else if (_globalTaskScope === 'created_by_me' && currentUserId) { params.set('created_by_concept_id', currentUserId); } + if (_selectedProjectId) { + params.set('project_concept_id', _selectedProjectId); + params.set('bulk_visibility', 'include'); + if (_selectedCollectionId) params.set('collection_concept_id', _selectedCollectionId); + return `/api/tasks/search?${params.toString()}`; + } return `/api/tasks/?${params.toString()}`; } @@ -606,6 +645,14 @@ function buildMyTasksUrl(statusFilter = null) { } async function ensureTaskTaxonomyLoaded() { + const generation = _globalTaskLoadGeneration; + try { + const projectResponse = await getJson('/api/tasks/projects'); + if (generation !== _globalTaskLoadGeneration) return; + _taskProjects = Array.isArray(projectResponse?.projects) ? projectResponse.projects : []; + } catch (err) { + console.debug('[taskPanel] Failed to load projects:', err); + } try { const taxonomy = await getJson('/api/tasks/taxonomy'); if (taxonomy && typeof taxonomy === 'object') { @@ -666,6 +713,7 @@ function getTaskDetailState(taskId) { task: null, comments: [], attachments: [], + worklog: [], history: [], }; } @@ -880,6 +928,14 @@ function renderGlobalTasksTabContent() {

Board lanes, Vontology-backed task categories, and a richer detail view live together here.

+ + + +
`; +} + function renderTaskHistoryRows(history) { if (!Array.isArray(history) || history.length === 0) { return '

No history events

'; @@ -2480,7 +2591,7 @@ function renderTaskHistoryRows(history) { return history.slice(0, 20).map((eventRow) => `
  • ${escapeHtml(eventRow.event_type || 'event')}
    -
    ${escapeHtml(formatDateTimeOrDash(eventRow.created_at))}
    +
    ${escapeHtml(formatDateTimeOrDash(eventRow.timestamp || eventRow.created_at))}
    • `).join(''); } @@ -2757,6 +2868,7 @@ function renderTaskDetailsPanel(task, detailState) { + @@ -2766,6 +2878,7 @@ function renderTaskDetailsPanel(task, detailState) { ${renderTaskHistoryRows(detailState.history)} + ${renderRetainedSourceAndWorklog(taskId, detailState)} `; } @@ -2798,7 +2911,7 @@ function renderTaskTimeline(detailState, task) {
    ${escapeHtml((step?.event_type || 'event').replace(/_/g, ' '))} - ${escapeHtml(formatDateTimeOrDash(step?.created_at))} + ${escapeHtml(formatDateTimeOrDash(step?.timestamp || step?.created_at))}
    `).join('')} @@ -3061,6 +3174,7 @@ function renderTaskInspector(task, detailState) { + @@ -3070,6 +3184,7 @@ function renderTaskInspector(task, detailState) { ${renderTaskHistoryRows(detailState?.history)} + ${renderRetainedSourceAndWorklog(taskId, detailState)} `; } @@ -3299,17 +3414,19 @@ async function loadTaskDetails(taskId, { refreshList = false } = {}) { renderTaskList(); try { - const [task, commentsResult, attachmentsResult, historyResult] = await Promise.all([ + const [task, commentsResult, attachmentsResult, historyResult, worklogResult] = await Promise.all([ getJson(`/api/tasks/${encodeURIComponent(taskId)}`), getJson(`/api/tasks/${encodeURIComponent(taskId)}/comments?limit=100`), getJson(`/api/tasks/${encodeURIComponent(taskId)}/attachments?limit=100`), getJson(`/api/tasks/${encodeURIComponent(taskId)}/history?limit=200`), + getJson(`/api/tasks/${encodeURIComponent(taskId)}/worklog?limit=200`), ]); if (scopeGenerationAtStart !== _taskQueueActivityGeneration) return; detailState.task = task; detailState.comments = commentsResult.comments || []; detailState.attachments = attachmentsResult.attachments || []; detailState.history = historyResult.history || []; + detailState.worklog = worklogResult.worklog || []; replaceCachedTask(task); if (refreshList) { updateTaskCountBadge(); @@ -3435,8 +3552,26 @@ async function addTaskAttachment(taskId, panelEl) { const uri = (panelEl.querySelector('.task-attachment-uri-input')?.value || '').trim(); const note = (panelEl.querySelector('.task-attachment-note-input')?.value || '').trim(); + const file = panelEl.querySelector('.task-attachment-file-input')?.files?.[0]; + if (file) { + if (file.size > 100 * 1024 * 1024) { + showToast('Files must be no larger than 100 MiB', 'warning'); + return; + } + const form = new FormData(); + form.append('file', file); + form.append('note', note); + const response = await fetch(`/api/tasks/${encodeURIComponent(taskId)}/attachments`, { + method: 'POST', body: form, + headers: { [WINDOW_SESSION_HEADER]: await ensureUniqueWindowSessionId() }, + }); + if (!response.ok) throw new Error('Attachment upload failed'); + await loadTaskDetails(taskId, { refreshList: true }); + showToast('Attachment uploaded', 'success'); + return; + } if (!filename || !uri) { - showToast('Filename and URI are required', 'warning'); + showToast('Choose a file, or enter a filename and URI', 'warning'); return; } @@ -3595,6 +3730,28 @@ function attachTaskEventListeners() { }); }); + root.querySelectorAll('.task-add-worklog-btn').forEach((button) => { + button.addEventListener('click', async () => { + const panel = button.closest('.task-detail-panel') || button.closest('.task-inspector-card'); + const minutes = Number(panel?.querySelector('.task-worklog-minutes')?.value); + if (!Number.isInteger(minutes) || minutes <= 0) { + showToast('Enter a positive number of minutes', 'error'); + return; + } + button.disabled = true; + try { + await postJson(`/api/tasks/${encodeURIComponent(button.dataset.taskId)}/worklog`, { + time_spent_minutes: minutes, + comment: panel.querySelector('.task-worklog-comment')?.value || '', + }); + await loadTaskDetails(button.dataset.taskId); + } catch (err) { + showToast('Could not save time entry', 'error'); + button.disabled = false; + } + }); + }); + root.querySelectorAll('.task-add-attachment-btn').forEach((btn) => { btn.addEventListener('click', async (e) => { const taskId = e.currentTarget.dataset.taskId; @@ -3755,7 +3912,7 @@ async function updateTaskStatus(taskId, newStatus) { try { await patchJson(`/api/tasks/${encodeURIComponent(taskId)}`, { status: newStatus }); showToast('Task updated', 'success'); - await loadTasks(_currentSessionId); + await refreshTasks(); } catch (err) { console.error('[taskPanel] Failed to update task:', err); showToast('Failed to update task', 'error'); @@ -3834,14 +3991,14 @@ export function getTasks() { /** * Refresh tasks for the current context. - * If a conversation session is active, loads tasks for that conversation. - * Otherwise, loads all tasks for the current user (global view). + * The visible global workspace takes precedence over a background conversation. + * Otherwise, refresh the conversation panel or current user's tasks. */ export async function refreshTasks() { - if (_currentSessionId) { - await loadTasks(_currentSessionId); - } else if (_isGlobalTabMode) { + if (_isGlobalTabMode) { await loadGlobalTasks(); + } else if (_currentSessionId) { + await loadTasks(_currentSessionId); } else { await loadMyTasks(); } diff --git a/tests/backend/test_internal_mcp_task_parity_tools.py b/tests/backend/test_internal_mcp_task_parity_tools.py index 2ea3da80d..6b2b74cd5 100644 --- a/tests/backend/test_internal_mcp_task_parity_tools.py +++ b/tests/backend/test_internal_mcp_task_parity_tools.py @@ -977,6 +977,8 @@ def _fake_create_task( reference_code=None, agent_creation_fingerprint=None, agent_creation_request_id=None, + project_concept_id=None, + collection_concept_ids=None, ): return { "task_concept_id": "#V#task_123", diff --git a/tests/backend/test_jira_cutover_integrity.py b/tests/backend/test_jira_cutover_integrity.py new file mode 100644 index 000000000..133569a23 --- /dev/null +++ b/tests/backend/test_jira_cutover_integrity.py @@ -0,0 +1,362 @@ +"""Outcome tests for interrupted migration, attachment integrity and cutover.""" + +import base64 +import copy +import hashlib +from datetime import UTC, datetime +from types import SimpleNamespace + +import pytest + +from src.backend.services import jira_source_retention_service as retention +from src.backend.services import jira_task_import_service as importer +from src.backend.services import jira_task_reconciliation_service as reconciliation +from src.backend.services import task_management_service as tasks +from src.backend.services import task_project_service as projects + + +def test_interrupted_activity_append_returns_original_without_duplicate_history( + monkeypatch, +): + original = { + "comment_id": "original", + "body": "preserve me", + "source": {"source_system": "jira", "external_id": "123"}, + } + doc = {"metadata": {"comments": [original]}} + monkeypatch.setattr(tasks, "_get_task_doc", lambda key: (key, doc)) + queries = [] + monkeypatch.setattr( + tasks.ConceptsRepository, + "update_one", + lambda query, update: ( + queries.append(query) or SimpleNamespace(matched_count=0) + ), + ) + monkeypatch.setattr( + tasks, + "_append_task_history_event", + lambda **kwargs: pytest.fail("duplicate event"), + ) + result = tasks.add_task_comment( + "#V#task", body="preserve me", source=original["source"] + ) + assert result == original + assert queries[0]["metadata.comments"]["$not"]["$elemMatch"] == { + "source.source_system": "jira", + "source.external_id": "123", + } + + +def test_dates_compare_by_instant_and_cross_batch_relations_do_not_fetch_jira( + monkeypatch, +): + assert importer._projection_value( + "due_date", datetime(2026, 9, 10, tzinfo=UTC) + ) == importer._projection_value("due_date", "2026-09-10T12:00:00+12:00") + calls = [] + monkeypatch.setattr( + importer, + "set_task_parent", + lambda child, parent, **kw: calls.append((child, parent)), + ) + monkeypatch.setattr( + importer, + "find_task_by_external_reference", + lambda **kw: pytest.fail("target should be in retained rows"), + ) + rows = [ + { + "jira_issue_key": "KKAT-2", + "task_concept_id": "#V#child", + "parent_issue_key": "KKAT-1", + "action": "created", + }, + { + "jira_issue_key": "KKAT-1", + "task_concept_id": "#V#parent", + "action": "created", + }, + ] + report = importer.repair_jira_task_relations(rows, actor_concept_id="#V#owner") + assert calls == [("#V#child", "#V#parent")] + assert report == {"mapped_relations": 1, "dropped_relations": 0} + + +def test_refresh_preserves_native_hierarchy_edits(monkeypatch): + monkeypatch.setattr( + importer, "get_task", lambda key: {"parent_task_concept_id": "#V#native_parent"} + ) + monkeypatch.setattr( + importer, + "set_task_parent", + lambda *args, **kw: pytest.fail("Native parent must survive"), + ) + rows = [ + { + "jira_issue_key": "KKAT-2", + "task_concept_id": "#V#child", + "parent_issue_key": "KKAT-1", + "guard_existing_hierarchy": True, + "hierarchy_baseline": {"parent": "KKAT-1"}, + "action": "updated", + }, + { + "jira_issue_key": "KKAT-1", + "task_concept_id": "#V#source_parent", + "action": "updated", + }, + ] + result = importer.repair_jira_task_relations(rows, actor_concept_id="#V#owner") + assert result["dropped_relations"] == 1 + assert "Native parent edit preserved" in rows[0]["relation_results"][0]["reason"] + + +@pytest.mark.asyncio +async def test_enhanced_board_pagination_requires_progress_and_terminal_evidence(): + tokens = [] + + class Source: + async def get_migration_resource(self, **kwargs): + tokens.append(kwargs["next_page_token"]) + if len(tokens) == 1: + return { + "issues": [{"id": "1"}], + "nextPageToken": "next", + "isLast": False, + } + return {"issues": [{"id": "2"}], "isLast": True} + + result = await retention.capture_resource( + Source(), "board_issues", "12", item_key="issues" + ) + assert result["complete"] and result["count"] == 2 + assert tokens == [None, "next"] + + class Stalled: + async def get_migration_resource(self, **kwargs): + return {"issues": [{"id": "1"}], "nextPageToken": "same", "isLast": False} + + result = await retention.capture_resource( + Stalled(), "board_issues", "12", item_key="issues" + ) + assert not result["complete"] + + +def test_attachment_upload_is_stored_and_read_back_before_linking(monkeypatch): + from src.backend.services import computer_file_copy_service as files + + monkeypatch.setattr(tasks, "_get_task_doc", lambda key: (key, {"metadata": {}})) + writes = [] + monkeypatch.setattr( + files, + "import_bytes_file_copy", + lambda **kw: (writes.append(kw) or {"success": True, "concept_id": "#V#file"}), + ) + monkeypatch.setattr( + files, + "fetch_file_copy_bytes", + lambda **kw: {"success": True, "data": b"original"}, + ) + monkeypatch.setattr(tasks, "add_task_attachment", lambda key, **kw: kw) + saved = tasks.add_task_attachment_bytes( + "#V#task", + data=b"original", + filename="evidence.txt", + actor_concept_id="#V#owner", + ) + assert saved["file_copy_concept_id"] == "#V#file" + assert saved["source"]["sha256"] == hashlib.sha256(b"original").hexdigest() + assert writes[0]["visibility_scope_mode"] == "user_only_default" + monkeypatch.setattr( + files, + "fetch_file_copy_bytes", + lambda **kw: {"success": True, "data": b"corrupted"}, + ) + with pytest.raises(tasks.TaskManagementError, match="read-back"): + tasks.add_task_attachment_bytes( + "#V#task", + data=b"original", + filename="evidence.txt", + actor_concept_id="#V#owner", + ) + + +def test_imported_assignment_is_provenance_without_a_new_audience(monkeypatch): + monkeypatch.setattr( + tasks, "_get_task_doc", lambda key: (key, {"relationships": {}}) + ) + edges = [] + monkeypatch.setattr( + tasks.ConceptsRepository, + "mutate_relationship_edge", + lambda **kw: edges.append(kw), + ) + monkeypatch.setattr(tasks.ConceptsRepository, "update_one", lambda *args: None) + monkeypatch.setattr(tasks, "_append_task_history_event", lambda **kw: None) + monkeypatch.setattr(tasks, "get_task", lambda key: {"task_concept_id": key}) + tasks.assign_task( + "#V#task", "#V#historical_source_assignee", update_visibility=False + ) + assert edges and {edge["kind"] for edge in edges} == {tasks.PREDICATE_HAS_ASSIGNEE} + + +def test_native_subtask_stays_in_its_parents_project_and_collections(monkeypatch): + metadata = { + "project_concept_id": "#V#project", + "collection_concept_ids": ["#V#collection"], + "organisation_concept_id": None, + } + monkeypatch.setattr( + tasks, "_get_task_doc", lambda key: (key, {"metadata": metadata}) + ) + created = [] + monkeypatch.setattr( + tasks, + "create_task", + lambda **kwargs: (created.append(kwargs) or {"task_concept_id": "#V#child"}), + ) + monkeypatch.setattr( + tasks, + "set_task_parent", + lambda child, parent, **kwargs: { + "task_concept_id": child, + "parent_task_concept_id": parent, + }, + ) + result = tasks.create_subtask( + "#V#parent", title="Child", description="Native project task" + ) + assert created[0]["project_concept_id"] == "#V#project" + assert created[0]["collection_concept_ids"] == ["#V#collection"] + assert result["task"]["parent_task_concept_id"] == "#V#parent" + + +def test_reconciliation_detects_wrong_relation_and_corrupt_retained_binary(monkeypatch): + source = [ + { + "key": "KKAT-1", + "id": "1", + "fields": { + "project": {"key": "KKAT"}, + "updated": "now", + "parent": {"key": "OTHER-1"}, + "issuetype": {"name": "Document"}, + }, + } + ] + project = { + "concept_id": "#V#project", + "key": "KKAT", + "source_id": "12", + "collection_concept_ids": ["#V#collection"], + "source_archive": {"complete": True, "file_copy_concept_id": "project"}, + } + task = { + "concept_id": "#V#task", + "relationships": {}, + "metadata": { + "project_concept_id": "#V#project", + "collection_concept_ids": ["#V#collection"], + "external_references": { + "jira": { + "external_id": "KKAT-1", + "issue_id": "1", + "updated": "now", + "source_archive": { + "complete": True, + "file_copy_concept_id": "issue", + }, + } + }, + }, + } + target = { + "concept_id": "#V#other", + "metadata": {"external_references": {"jira": {"external_id": "OTHER-1"}}}, + } + from src.backend.db.repositories.concepts_repository import ConceptsRepository + + monkeypatch.setattr( + ConceptsRepository, + "find", + lambda query, **kw: [task] if "$or" in query else [target], + ) + monkeypatch.setattr(projects, "get_task_project", lambda key: project) + monkeypatch.setattr( + projects, + "get_task_collection", + lambda key: { + "concept_id": key, + "project_concept_ids": ["#V#project"], + "selection": {"kind": "project_membership"}, + }, + ) + + def archive(ref, **kwargs): + if ref["file_copy_concept_id"] == "project": + return {"complete": True, "source": {"id": "12"}} + return { + "complete": True, + "source": source[0], + "binaries": [ + { + "id": "file", + "complete": True, + "source": {"content_base64": base64.b64encode(b"wrong").decode()}, + "size_bytes": 5, + "sha256": hashlib.sha256(b"right").hexdigest(), + } + ], + } + + monkeypatch.setattr(retention, "read_source_archive", archive) + result = reconciliation.reconcile_retained_project( + source_issues=source, + project_concept_id="#V#project", + actor_concept_id="#V#owner", + ) + assert not result["retention_reconciled"] + assert {p["kind"] for p in result["problems"]} == { + "archive_readback_failed", + "relation_mismatch", + } + + +def test_cutover_requires_all_material_receipts_and_readback(monkeypatch): + project = {"concept_id": "#V#project", "writer": "jira"} + monkeypatch.setattr( + projects, "get_task_project", lambda key: copy.deepcopy(project) + ) + + def update(key, fields): + for name, value in fields.items(): + project[name.rsplit(".", 1)[-1]] = value + + monkeypatch.setattr(projects, "update_concept", update) + evidence = { + "reconciliation": { + "project_concept_id": "#V#project", + "retention_reconciled": True, + } + } + with pytest.raises(ValueError, match="receipts"): + projects.set_task_project_writer( + "#V#project", writer="von", actor_concept_id="#V#owner", evidence=evidence + ) + assert project["writer"] == "jira" + evidence.update( + { + key: {"receipt": key} + for key in ( + "site_retention_receipt", + "ordinary_use_receipt", + "restore_receipt", + "final_delta_receipt", + ) + } + ) + saved = projects.set_task_project_writer( + "#V#project", writer="von", actor_concept_id="#V#owner", evidence=evidence + ) + assert saved["writer"] == "von" and saved["writer_history"][0]["from"] == "jira" diff --git a/tests/backend/test_jira_read_authority.py b/tests/backend/test_jira_read_authority.py index af0cb6b7d..a8cd421a3 100644 --- a/tests/backend/test_jira_read_authority.py +++ b/tests/backend/test_jira_read_authority.py @@ -244,12 +244,21 @@ def test_operator_migration_cli_binds_authority_only_around_its_runner( ) monkeypatch.setattr( - cli, "_parse_args", lambda: SimpleNamespace(report_path=Path("unused.json")) + cli, + "_parse_args", + lambda: SimpleNamespace( + report_path=Path("unused.json"), + actor_concept_id="#V#operator", + organisation_concept_id=None, + ), ) calls = [] def run(_options): + from src.backend.security.access_control import get_effective_user_concept_id + assert internal_mcp_actor_context_is_trusted_local_operator() + assert get_effective_user_concept_id() == "#V#operator" calls.append(True) return {"dry_run": True} diff --git a/tests/backend/test_jira_source_retention_service.py b/tests/backend/test_jira_source_retention_service.py new file mode 100644 index 000000000..6ecb4a760 --- /dev/null +++ b/tests/backend/test_jira_source_retention_service.py @@ -0,0 +1,182 @@ +import base64 +import copy +import gzip +import hashlib +import json + +import pytest + +from src.backend.integrations.internal_mcp.jira_migration_resources import ( + resource_request, +) +from src.backend.services import jira_source_retention_service as retention + + +class Source: + def __init__(self): + self.original = { + "id": "123", + "key": "KKAT-1", + "fields": { + "updated": "2026-09-10", + "customfield_42": {"unknown": [None, " Māori\n", 12]}, + "description": { + "type": "doc", + "content": [{"type": "table", "attrs": {"layout": "wide"}}], + }, + "attachment": [{"id": "45", "size": 3, "filename": "test.bin"}], + }, + } + + async def get_issue(self, **kwargs): + return copy.deepcopy(self.original) + + async def get_migration_resource(self, resource, start_at=0, **kwargs): + key = { + "comments": "comments", + "worklogs": "worklogs", + "changelog": "values", + }.get(resource) + if key: + # Server pages smaller than requested, including rich activity. + return { + "startAt": start_at, + "maxResults": 1, + "total": 2, + key: [ + { + "id": str(start_at), + "body": { + "type": "doc", + "content": [{"type": "text", "text": " x\n"}], + }, + } + ], + } + return {"keys": []} + + async def get_watchers(self, **kwargs): + return {"watchCount": 0, "watchers": []} + + async def get_attachment_content(self, **kwargs): + return { + "success": True, + "content_base64": base64.b64encode(b"\x00\x01\xff").decode(), + } + + +@pytest.mark.asyncio +async def test_capture_preserves_original_and_all_pages_and_binary(): + source = Source() + expected = copy.deepcopy(source.original) + archive, projection = await retention.capture_issue(source, "KKAT-1") + assert archive["complete"] + assert archive["source"] == source.original == expected + assert archive["resources"]["comments"]["count"] == 2 + assert len(archive["resources"]["comments"]["pages"]) == 2 + assert len(projection["changelog"]["histories"]) == 2 + assert ( + archive["binaries"][0]["sha256"] == hashlib.sha256(b"\x00\x01\xff").hexdigest() + ) + assert "content_base64" not in archive["source"]["fields"]["attachment"][0] + assert ( + json.loads(gzip.decompress(gzip.compress(retention.canonical_json(archive)))) + == archive + ) + + +@pytest.mark.asyncio +async def test_issue_resources_use_stable_id_when_key_route_is_unavailable(): + class StableIdSource(Source): + async def get_migration_resource(self, resource, identifier, **kwargs): + if resource == "worklogs" and identifier != "123": + return {"success": False, "status_code": 404, "error": "not_found"} + return await super().get_migration_resource(resource, **kwargs) + + archive, _ = await retention.capture_issue(StableIdSource(), "KKAT-1") + assert archive["complete"] + assert archive["resources"]["worklogs"]["identifier"] == "123" + assert archive["resources"]["worklogs"]["count"] == 2 + + +@pytest.mark.asyncio +async def test_duplicate_page_and_unavailable_binary_cannot_claim_retention(): + source = Source() + + async def pages(**kwargs): + key = { + "comments": "comments", + "worklogs": "worklogs", + "changelog": "values", + }.get(kwargs["resource"]) + return {key: [{"id": "repeated"}], "total": 3} if key else {"keys": []} + + async def attachment(**kwargs): + return {"success": False, "error": "forbidden"} + + source.get_migration_resource = pages + source.get_attachment_content = attachment + archive, _ = await retention.capture_issue(source, "KKAT-1") + assert not archive["complete"] + assert any( + error["error"] == "duplicate_source_item" + for error in archive["resources"]["comments"]["errors"] + ) + assert not archive["binaries"][0]["complete"] + + +@pytest.mark.asyncio +async def test_missing_terminal_contract_is_not_a_complete_empty_page(): + source = Source() + + async def pages(**kwargs): + return {"values": []} + + source.get_migration_resource = pages + result = await retention.capture_resource( + source, "changelog", "KKAT-1", item_key="values" + ) + assert not result["complete"] + + +def test_resource_paths_cannot_redirect_credentials(): + path, _ = resource_request( + "project_property", "https://evil.example/a", "../secrets" + ) + assert path.startswith("/rest/api/3/project/https%3A%2F%2Fevil.example%2Fa/") + assert "../" not in path + with pytest.raises(ValueError): + resource_request("https://evil.example") + with pytest.raises(ValueError): + resource_request("project", "..") + + +def test_archive_read_requires_access_and_verifies_bytes(monkeypatch): + from src.backend.services import computer_file_copy_service as files + + raw = retention.canonical_json({"kind": "issue", "source": {"key": "KKAT-1"}}) + data = gzip.compress(raw, mtime=0) + ref = { + "file_copy_concept_id": "#V#file", + "content_sha256": hashlib.sha256(raw).hexdigest(), + "blob_sha256": hashlib.sha256(data).hexdigest(), + } + monkeypatch.setattr( + files, + "fetch_file_copy_bytes", + lambda **kw: ( + {"success": True, "data": data} + if kw["user_concept_id"] == "#V#owner" + else {"success": False} + ), + ) + assert ( + retention.read_source_archive(ref, actor_concept_id="#V#owner")["source"]["key"] + == "KKAT-1" + ) + with pytest.raises(ValueError, match="unavailable"): + retention.read_source_archive(ref, actor_concept_id="#V#other") + with pytest.raises(ValueError, match="checksum"): + retention.read_source_archive( + {**ref, "blob_sha256": "wrong"}, actor_concept_id="#V#owner" + ) diff --git a/tests/backend/test_jira_task_import_service.py b/tests/backend/test_jira_task_import_service.py index 571b7cdd4..29b131c49 100644 --- a/tests/backend/test_jira_task_import_service.py +++ b/tests/backend/test_jira_task_import_service.py @@ -761,3 +761,173 @@ def _fake_upsert_task_external_reference( assert imported_activity["attachment_ids"] == ["20001"] assert imported_activity["worklog_ids"] == ["30001"] assert len(imported_activity["status_history_signatures"]) == 1 + + +def test_retention_sync_preserves_conflicting_von_edits_and_originals(monkeypatch): + existing = { + "task_concept_id": "#V#task", + "title": "Native edit", + "description": "Example description", + "status": "pending", + "priority": "medium", + "labels": [], + "external_references": { + "jira": { + "last_import_projection": { + "title": "Earlier source", + "description": "Example description", + "status": "pending", + "priority": "medium", + "labels": [], + }, + "source_archive": { + "content_sha256": "old", + "file_copy_concept_id": "#V#old", + }, + } + }, + } + updates, references = [], [] + monkeypatch.setattr( + import_service, "find_task_by_external_reference", lambda **kw: existing + ) + monkeypatch.setattr( + import_service, + "update_task_fields", + lambda task, **kw: updates.append(kw["fields"]), + ) + monkeypatch.setattr( + import_service, "_materialise_jira_issue_activity", lambda **kw: ({}, [], []) + ) + monkeypatch.setattr( + import_service, + "upsert_task_external_reference", + lambda task, **kw: references.append(kw["reference_payload"]), + ) + issue = _jira_issue("KKAT-1", summary="New source", status="Done") + issue["_von_source_archive"] = { + "content_sha256": "new", + "file_copy_concept_id": "#V#new", + } + result = import_service.import_jira_issues_to_tasks(issues=[issue], dry_run=False) + assert "title" not in updates[0] + assert updates[0]["status"] == "completed" + assert result["issues"][0]["sync_conflicts"] == [ + {"field": "title", "resolution": "preserved_von_value"} + ] + assert references[0]["source_archive_history"][0]["content_sha256"] == "old" + assert references[0]["source_archive"]["content_sha256"] == "new" + + +def test_legacy_recorded_watcher_projection_allows_source_refresh(monkeypatch): + existing = { + "task_concept_id": "#V#task", + "title": "Example summary", + "description": "Example description", + "status": "pending", + "priority": "medium", + "labels": [], + "watcher_concept_ids": [], + "external_references": { + "jira": { + "status_name": "To Do", + "priority_name": "Medium", + "labels": [], + "jira_watcher_concept_ids": [], + } + }, + } + updates = [] + monkeypatch.setattr( + import_service, "find_task_by_external_reference", lambda **kw: existing + ) + monkeypatch.setattr( + import_service, + "update_task_fields", + lambda task, **kw: updates.append(kw["fields"]), + ) + monkeypatch.setattr( + import_service, "_materialise_jira_issue_activity", lambda **kw: ({}, [], []) + ) + monkeypatch.setattr( + import_service, "upsert_task_external_reference", lambda *args, **kw: None + ) + issue = _jira_issue("KKAT-1", watcher_account_ids=["watcher"]) + issue["_von_source_archive"] = { + "content_sha256": "new", + "file_copy_concept_id": "#V#new", + } + result = import_service.import_jira_issues_to_tasks( + issues=[issue], + dry_run=False, + jira_account_id_to_concept_id={"watcher": "#V#watcher"}, + ) + assert updates[0]["watcher_concept_ids"] == ["#V#watcher"] + assert not result["issues"][0]["sync_conflicts"] + + +def test_jira_refresh_cannot_write_von_authoritative_task(monkeypatch): + from src.backend.services import task_project_service + + existing = {"task_concept_id": "#V#task", "project_concept_id": "#V#project"} + monkeypatch.setattr( + import_service, "find_task_by_external_reference", lambda **kw: existing + ) + monkeypatch.setattr( + task_project_service, "get_task_project", lambda key: {"writer": "von"} + ) + + def unexpected(*args, **kwargs): + raise AssertionError( + "Von-authoritative task must not be changed by Jira refresh" + ) + + for name in ( + "update_task_fields", + "create_task", + "upsert_task_external_reference", + "set_task_parent", + "_materialise_jira_issue_activity", + ): + monkeypatch.setattr(import_service, name, unexpected) + report = import_service.import_jira_issues_to_tasks( + issues=[_jira_issue("KKAT-1", parent_key="KKAT-2")], dry_run=False + ) + assert report["issues"][0]["action"] == "skipped_von_authoritative" + + +def test_participant_preview_and_repeated_cache_resolution_do_not_write_aliases( + monkeypatch, +): + writes = [] + monkeypatch.setattr( + import_service, + "_upsert_jira_participant_aliases", + lambda **kw: writes.append(kw), + ) + kwargs = dict( + raw_participant={"accountId": "jira-person", "displayName": "Person"}, + account_id_to_concept_id={"jira-person": "#V#person"}, + account_cache={}, + email_cache={}, + actor_concept_id="#V#owner", + organisation_concept_id=None, + allow_lookup=False, + allow_create=False, + ) + import_service._ensure_jira_participant_concept(**kwargs, allow_alias_writes=False) + assert not writes + kwargs["account_id_to_concept_id"] = {} + import_service._ensure_jira_participant_concept(**kwargs) + assert not writes + + +def test_custom_status_retains_source_name_and_uses_jira_category(): + assert import_service._map_status("Candidate declined", "done") == ( + "completed", + None, + ) + assert import_service._map_status("Interviewing", "indeterminate") == ( + "in_progress", + None, + ) diff --git a/tests/backend/test_jira_task_migration_runner_service.py b/tests/backend/test_jira_task_migration_runner_service.py index c51b7f93f..42fd941e0 100644 --- a/tests/backend/test_jira_task_migration_runner_service.py +++ b/tests/backend/test_jira_task_migration_runner_service.py @@ -189,3 +189,75 @@ async def _fake_get_jira_proxy(): assert [item["key"] for item in gateway.calls[2]["payload"]["issue_keys"]] == [ "JVNAUTOSCI-1407" ] + + +def test_failed_batch_is_checkpointed_and_resume_reuses_completed_work(tmp_path): + import json + import pytest + from dataclasses import replace + + docs = [ + {"id": str(i), "key": f"KKAT-{i}", "fields": {"updated": "v1"}} for i in (1, 2) + ] + options = mod.JiraTaskMigrationOptions( + actor_concept_id="#V#owner", + project_key="KKAT", + batch_size=1, + passes=1, + preserve_source=True, + report_path=tmp_path / "report.json", + ) + first = {"success": True, "summary": {"created": 1}, "issues": []} + failed = { + "success": False, + "fetch": {"fetch_errors": [{"issue_key": "KKAT-2", "error": "unavailable"}]}, + } + with pytest.raises(RuntimeError, match="failed"): + mod._run_import_batches( + gateway=_FakeGateway([first, failed]), + issue_docs=docs, + options=options, + run_label="current_scope", + ) + checkpoint = json.loads( + (tmp_path / "report.current_scope.checkpoint.json").read_text() + ) + assert checkpoint["completed_batches"] == [first] + assert checkpoint["last_batch"] == failed + retry = _FakeGateway([first]) + report = mod._run_import_batches( + gateway=retry, + issue_docs=docs, + options=replace(options, resume=True), + run_label="current_scope", + ) + assert len(retry.calls) == 1 + assert retry.calls[0]["payload"]["issue_keys"][0]["key"] == "KKAT-2" + assert report["summary"]["created"] == 2 + changed = [{**docs[0], "fields": {"updated": "v2"}}, docs[1]] + with pytest.raises(ValueError, match="scope/source changed"): + mod._run_import_batches( + gateway=_FakeGateway([]), + issue_docs=changed, + options=replace(options, resume=True), + run_label="current_scope", + ) + + +def test_discovery_errors_cannot_be_reported_as_empty_project(monkeypatch): + import pytest + + class Proxy: + async def search(self, **kwargs): + return {"success": False, "status_code": 403} + + async def get_proxy(): + return Proxy() + + monkeypatch.setattr(mod, "get_jira_proxy", get_proxy) + with pytest.raises(RuntimeError, match="not an empty project"): + asyncio.run( + mod.discover_jira_issue_docs( + jql="project = KKAT", fields=["key"], page_size=100 + ) + ) diff --git a/tests/backend/test_mcp_stdio_coding_agent_access_profile.py b/tests/backend/test_mcp_stdio_coding_agent_access_profile.py index 6703d86c9..d598f7173 100644 --- a/tests/backend/test_mcp_stdio_coding_agent_access_profile.py +++ b/tests/backend/test_mcp_stdio_coding_agent_access_profile.py @@ -156,7 +156,7 @@ async def _stub_handler(arguments): mcp_stdio_server._json_text( { "success": True, - "tool": "upsert_text_relation", + "tool": "assign_task", "arguments": arguments, } ) @@ -164,13 +164,13 @@ async def _stub_handler(arguments): monkeypatch.setitem( mcp_stdio_server._TOOL_HANDLERS, - "upsert_text_relation", + "assign_task", _stub_handler, ) async def _runner(): result = await mcp_stdio_server.call_tool( - "upsert_text_relation", + "assign_task", {"concept_id": "#V#x", "predicate": "#V#hasNote", "text": "hello"}, ) return _decode_text_payload(result) @@ -178,7 +178,7 @@ async def _runner(): payload = asyncio.run(_runner()) assert payload["success"] is True - assert payload["tool"] == "upsert_text_relation" + assert payload["tool"] == "assign_task" assert payload["arguments"]["text"] == "hello" @@ -197,7 +197,7 @@ async def _stub_handler(arguments): mcp_stdio_server._json_text( { "success": True, - "tool": "upsert_text_relation", + "tool": "assign_task", "arguments": arguments, } ) @@ -205,13 +205,13 @@ async def _stub_handler(arguments): monkeypatch.setitem( mcp_stdio_server._TOOL_HANDLERS, - "upsert_text_relation", + "assign_task", _stub_handler, ) async def _runner(): result = await mcp_stdio_server.call_tool( - "upsert_text_relation", + "assign_task", {"concept_id": "#V#x", "predicate": "#V#hasNote", "text": "hello"}, ) return _decode_text_payload(result) @@ -219,4 +219,4 @@ async def _runner(): payload = asyncio.run(_runner()) assert payload["success"] is True - assert payload["tool"] == "upsert_text_relation" + assert payload["tool"] == "assign_task" diff --git a/tests/backend/test_mcp_stdio_task_actor.py b/tests/backend/test_mcp_stdio_task_actor.py new file mode 100644 index 000000000..8c5d186e4 --- /dev/null +++ b/tests/backend/test_mcp_stdio_task_actor.py @@ -0,0 +1,141 @@ +import json + +import pytest + +from src.backend.integrations.internal_mcp.gateway import ( + bind_internal_mcp_actor_context_source, +) +from src.backend.mcp_server import mcp_stdio_server as server +from src.backend.security.access_control import ( + get_effective_organisation_concept_id, + get_effective_user_concept_id, +) + + +@pytest.mark.asyncio +async def test_configured_task_principal_needs_no_caller_identity(monkeypatch): + monkeypatch.setenv("VON_MCP_TASK_ACTOR_CONCEPT_ID", "#V#owner") + monkeypatch.setenv("VON_MCP_TASK_ORGANISATION_CONCEPT_ID", "#V#org") + + async def handler(arguments): + assert arguments["request_id"].startswith("task-mcp-") + assert get_effective_organisation_concept_id() == "#V#org" + return [ + server._json_text( + { + "success": True, + "actor": get_effective_user_concept_id(), + "creator": arguments["creator_concept_id"], + } + ) + ] + + monkeypatch.setitem(server._TOOL_HANDLERS, "task_get", handler) + original = {"task_concept_id": "KKAT-1"} + result = await server.call_tool("task_get", original) + assert json.loads(result[0].text) == { + "success": True, + "actor": "#V#owner", + "creator": "#V#owner", + } + assert original == {"task_concept_id": "KKAT-1"} + assert get_effective_user_concept_id() is None + + +@pytest.mark.asyncio +async def test_payload_cannot_impersonate_a_different_task_actor(monkeypatch): + monkeypatch.setenv("VON_MCP_TASK_ACTOR_CONCEPT_ID", "#V#owner") + + async def handler(arguments): + pytest.fail("Identity mismatch must be rejected before the task handler") + + monkeypatch.setitem(server._TOOL_HANDLERS, "task_get", handler) + result = await server.call_tool( + "task_get", + {"task_concept_id": "KKAT-1", "acting_user_concept_id": "#V#someone_else"}, + ) + assert json.loads(result[0].text)["error_code"] == "task_actor_mismatch" + + +@pytest.mark.asyncio +async def test_configured_operator_does_not_replace_an_inherited_task_context( + monkeypatch, +): + monkeypatch.setenv("VON_MCP_TASK_ACTOR_CONCEPT_ID", "#V#owner") + + async def handler(arguments): + assert "creator_concept_id" not in arguments + return [server._json_text({"success": True})] + + monkeypatch.setitem(server._TOOL_HANDLERS, "task_get", handler) + with bind_internal_mcp_actor_context_source("tool_payload_fallback"): + result = await server.call_tool("task_get", {"task_concept_id": "KKAT-1"}) + assert json.loads(result[0].text)["success"] + + +@pytest.mark.parametrize("organisation", [None, "#V#org"]) +def test_task_effects_support_personal_and_organisation_owners( + monkeypatch, organisation +): + from src.backend.integrations.internal_mcp import catalogue + from src.backend.security.access_control import override_current_actor + from src.backend.services import task_management_service as tasks + + monkeypatch.setattr( + tasks, + "get_task", + lambda key: { + "task_concept_id": key, + "created_by_concept_id": "#V#owner", + "organisation_concept_id": organisation, + }, + ) + with override_current_actor("#V#owner", organisation): + _scope, task, error = catalogue._authorise_task_actor_mutation( + {"task_concept_id": "#V#task"}, surface="task_add_comment" + ) + assert error is None and task["task_concept_id"] == "#V#task" + + +def test_local_task_operator_still_requires_canonical_task_access(monkeypatch): + from src.backend.integrations.internal_mcp import catalogue, gateway + from src.backend.security.access_control import override_current_actor + from src.backend.services import task_management_service as tasks + + def denied(key): + raise tasks.TaskNotFoundError("Task not found") + + monkeypatch.setattr(tasks, "get_task", denied) + with ( + override_current_actor("#V#owner", None), + gateway.bind_internal_mcp_actor_context_source( + gateway.INTERNAL_MCP_TRUSTED_LOCAL_OPERATOR_SOURCE, + preexisting_actor_context=("#V#owner", None), + ), + ): + _scope, task, error = catalogue._authorise_task_actor_mutation( + {"task_concept_id": "#V#private_to_someone_else"}, + surface="task_add_comment", + ) + assert task is None and error["error_code"] == "NOT_FOUND" + + +def test_ordinary_task_effect_cannot_cross_organisation_or_owner(monkeypatch): + from src.backend.integrations.internal_mcp import catalogue + from src.backend.security.access_control import override_current_actor + from src.backend.services import task_management_service as tasks + + monkeypatch.setattr( + tasks, + "get_task", + lambda key: { + "task_concept_id": key, + "created_by_concept_id": "#V#other", + "organisation_concept_id": "#V#org", + }, + ) + with override_current_actor("#V#owner", None): + _, task, error = catalogue._authorise_task_actor_mutation( + {"task_concept_id": "#V#task"}, surface="task_add_comment" + ) + assert task is None and error["error_code"] == "task_actor_scope_denied" diff --git a/tests/backend/test_task_project_service.py b/tests/backend/test_task_project_service.py new file mode 100644 index 000000000..1f136b080 --- /dev/null +++ b/tests/backend/test_task_project_service.py @@ -0,0 +1,80 @@ +import copy + +import pytest + +from src.backend.services import task_project_service as projects + + +def test_project_collection_rerun_preserves_identity_and_writer(monkeypatch): + docs = {} + monkeypatch.setattr(projects, "can_access_concept", lambda key: True) + monkeypatch.setattr( + projects, + "get_concept_by_concept_id_exact", + lambda key: copy.deepcopy(docs.get(key)), + ) + + def create(**kwargs): + key = kwargs["concept_id"] + assert key not in docs + assert kwargs["created_by_concept_id"] == "#V#owner" + assert not kwargs["maintain_relationship_inverses"] + docs[key] = copy.deepcopy(kwargs) + return docs[key] + + def update(key, fields): + for path, value in fields.items(): + docs[key]["attributes"][path.split(".", 1)[1]] = copy.deepcopy(value) + + monkeypatch.setattr(projects, "create_concept", create) + monkeypatch.setattr(projects, "update_concept", update) + source = { + "id": "12", + "key": "KKAT", + "name": "KnowKat", + "self": "https://jira.example/rest/api/3/project/12", + } + first = projects.ensure_jira_project(source, actor_concept_id="#V#owner") + project_id = first["project_concept_id"] + docs[project_id]["attributes"]["task_project"]["writer"] = "von" + second = projects.ensure_jira_project( + {**source, "name": "KnowKat updated"}, actor_concept_id="#V#owner" + ) + assert len(docs) == 2 + assert second["project_concept_id"] == project_id + assert second["collection"]["project_concept_ids"] == [project_id] + assert second["project"]["writer"] == "von" + assert ( + projects.validate_task_membership(project_id, second["collection_concept_ids"])[ + 1 + ] + == second["collection_concept_ids"] + ) + + +def test_membership_cannot_select_inaccessible_or_wrong_project(monkeypatch): + monkeypatch.setattr(projects, "can_access_concept", lambda key: key != "#V#private") + monkeypatch.setattr( + projects, + "get_concept_by_concept_id_exact", + lambda key: { + "attributes": { + "task_project": {}, + "task_collection": {"project_concept_ids": ["#V#different"]}, + } + }, + ) + with pytest.raises(ValueError, match="not accessible"): + projects.validate_task_membership("#V#private", []) + with pytest.raises(ValueError, match="does not cover"): + projects.validate_task_membership("#V#project", ["#V#collection"]) + + +def test_private_view_does_not_break_shared_project_navigation(monkeypatch): + monkeypatch.setattr(projects, "can_access_concept", lambda key: key == "#V#shared") + monkeypatch.setattr( + projects, "get_task_collection", lambda key: {"concept_id": key} + ) + assert projects.list_project_collections( + {"collection_concept_ids": ["#V#private", "#V#shared"]} + ) == [{"concept_id": "#V#shared"}] diff --git a/tests/frontend/taskPanelGroupFilters.test.js b/tests/frontend/taskPanelGroupFilters.test.js index 4cd8a4274..91d4f1794 100644 --- a/tests/frontend/taskPanelGroupFilters.test.js +++ b/tests/frontend/taskPanelGroupFilters.test.js @@ -57,6 +57,62 @@ describe('task panel ontology-backed groups', () => { jest.restoreAllMocks(); }); + test('project changes replace in-flight loads and ignore late project details', async () => { + const { getJson, patchJson } = require(apiServiceModulePath); + let finishOldTasks; + let finishOldProject; + const oldTasks = new Promise((resolve) => { finishOldTasks = resolve; }); + const oldProject = new Promise((resolve) => { finishOldProject = resolve; }); + getJson.mockImplementation((url) => { + if (url === '/api/tasks/taxonomy') return Promise.resolve(buildTaxonomyResponse()); + if (url === '/api/tasks/projects') return Promise.resolve({ projects: [ + { concept_id: '#V#project_a', key: 'A', name: 'First project' }, + { concept_id: '#V#project_b', key: 'B', name: 'Second project' }, + ] }); + if (url === '/api/tasks/projects/%23V%23project_a') return oldProject; + if (url === '/api/tasks/projects/%23V%23project_b') return Promise.resolve({ collections: [ + { concept_id: '#V#collection_b', name: 'Second collection' }, + ] }); + if (url.startsWith('/api/tasks/?')) return oldTasks; + if (url.startsWith('/api/tasks/search?')) { + const params = new URL(url, 'https://von.test').searchParams; + expect(params.get('project_concept_id')).toBe('#V#project_b'); + expect(params.get('bulk_visibility')).toBe('include'); + return Promise.resolve({ tasks: [{ task_concept_id: '#V#new_task', title: 'Second project task', status: 'pending' }], has_more: false }); + } + return Promise.resolve({}); + }); + const { showGlobalTasks, setCurrentSession } = require(taskPanelModulePath); + const opening = showGlobalTasks(); + await flushRenderQueue(); + function selectProject(value) { + const select = document.querySelector('#globalTaskProjectFilter'); + select.value = value; + select.dispatchEvent(new Event('change', { bubbles: true })); + } + selectProject('#V#project_a'); + selectProject('#V#project_b'); + await flushRenderQueue(); + finishOldProject({ collections: [{ concept_id: '#V#collection_a', name: 'Wrong collection' }] }); + finishOldTasks({ tasks: [{ task_concept_id: '#V#old_task', title: 'Wrong project task', status: 'pending' }] }); + await opening; + await flushRenderQueue(); + expect(document.querySelector('#globalTaskProjectFilter').value).toBe('#V#project_b'); + expect(document.querySelector('#globalTaskCollectionFilter').textContent).toContain('Second collection'); + expect(document.querySelector('#globalTaskCollectionFilter').textContent).not.toContain('Wrong collection'); + expect(Array.from(document.querySelectorAll('.task-item .task-title')).map((el) => el.textContent.trim())).toEqual(['Second project task']); + await setCurrentSession('background-conversation'); + patchJson.mockResolvedValue({ success: true }); + const status = document.querySelector('.task-status-select'); + status.value = 'in_progress'; + status.dispatchEvent(new Event('change', { bubbles: true })); + await flushRenderQueue(); + expect(patchJson).toHaveBeenCalledWith('/api/tasks/%23V%23new_task', { status: 'in_progress' }); + expect(getJson.mock.calls.some(([url]) => url.startsWith('/api/tasks/my'))).toBe(false); + expect(getJson.mock.calls.some(([url]) => url.includes('session_id=background-conversation'))).toBe(false); + expect(document.querySelector('.task-item .task-title').textContent.trim()).toBe('Second project task'); + }); + test('derives groups from ontology-linked task parents and filters visible tasks', async () => { const { getJson } = require(apiServiceModulePath); getJson.mockImplementation((url) => { diff --git a/tests/test_backup_task_project_files.py b/tests/test_backup_task_project_files.py new file mode 100644 index 000000000..a0c7093a5 --- /dev/null +++ b/tests/test_backup_task_project_files.py @@ -0,0 +1,89 @@ +import hashlib +import json + +import pytest +from cryptography.fernet import Fernet + +from scripts import backup_task_project_files as backup + + +def test_restore_uses_canonical_file_identity_and_rejects_corrupt_content( + tmp_path, monkeypatch +): + from src.backend.services import computer_file_copy_service as files + from src.backend.services import concept_service + + monkeypatch.setenv("VON_DB_NAME", "test_restored_tasks") + monkeypatch.setenv("VON_BLOB_STORE_BACKEND", "s3") + cipher = Fernet(Fernet.generate_key()) + data = b"retained original attachment\x00\xff" + digest = hashlib.sha256(data).hexdigest() + entry = { + "file_copy_concept_id": "#V#file", + "sha256": digest, + "size_bytes": len(data), + "content_type": "application/octet-stream", + "original_filename": "original.bin", + } + manifest = { + "source_database": "von_db", + "complete": True, + "project_concept_ids": ["#V#project"], + "task_count": 1, + "files": [entry], + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + encrypted = tmp_path / f"{digest}.bin.enc" + encrypted.write_bytes(cipher.encrypt(data)) + writes = [] + + def update(identifier, fields): + writes.append((identifier, fields)) + + monkeypatch.setattr(concept_service, "update_concept", update) + + def fetch(**kwargs): + from src.backend.services.blob_store import get_blob_store_from_env + + assert kwargs["file_copy_concept_id"] == "#V#file" + data = get_blob_store_from_env().get_bytes(writes[-1][1]["attributes.blob_key"]) + return {"success": True, "data": data} + + monkeypatch.setattr(files, "fetch_file_copy_bytes", fetch) + receipt = backup.restore_files( + backup_dir=tmp_path, + destination=tmp_path / "restored", + actor_id="#V#owner", + cipher=cipher, + ) + assert receipt["complete"] and receipt["restored_file_copy_ids"] == ["#V#file"] + assert not any("visibility" in key or "specific_to" in key for key in writes[0][1]) + encrypted.write_bytes(cipher.encrypt(b"corrupt")) + with pytest.raises(ValueError, match="integrity mismatch"): + backup.restore_files( + backup_dir=tmp_path, + destination=tmp_path / "other", + actor_id="#V#owner", + cipher=cipher, + ) + assert len(writes) == 1 + monkeypatch.setenv("VON_DB_NAME", "von_db") + with pytest.raises(ValueError, match="separate restored database"): + backup.restore_files( + backup_dir=tmp_path, + destination=tmp_path / "other", + actor_id="#V#owner", + cipher=cipher, + ) + + +def test_file_inventory_includes_archive_history_and_native_attachments(): + value = { + "source_archive": {"file_copy_concept_id": "#V#current"}, + "source_archive_history": [{"file_copy_concept_id": "#V#old"}], + "attachments": [ + {"file_copy_concept_id": "#V#attachment"}, + {"uri": "https://example.org"}, + ], + } + assert backup.file_copy_ids(value) == {"#V#current", "#V#old", "#V#attachment"} diff --git a/tests/test_restore_von_db.py b/tests/test_restore_von_db.py index a5c57d32c..71558914f 100644 --- a/tests/test_restore_von_db.py +++ b/tests/test_restore_von_db.py @@ -231,3 +231,27 @@ def test_dry_run_from_encrypted_zip_uses_fernet_key( exit_code = restore_von_db.main(["--backup-path", str(encrypted_path)]) assert exit_code == 0 + + +def test_legacy_von_prelude_does_not_shadow_mongodb_metadata(tmp_path, monkeypatch): + backup_root = _make_dump_tree(tmp_path) + original = json.dumps( + {"db_name": "von_db", "mongo_uri_redacted": "mongodb://example"} + ) + (backup_root / "prelude.json").write_text(original) + + def restore(command, **kwargs): + staged = Path(command[command.index("--dir") + 1]) + assert not (staged / "prelude.json").exists() + assert (staged / "von_db" / "papers.bson").read_bytes() == b"paper-data" + assert command[command.index("--nsTo") + 1] == "scratch.*" + + monkeypatch.setattr(restore_von_db.subprocess, "run", restore) + restore_von_db._run_mongorestore( + mongo_uri="mongodb://localhost/", + dump_root=backup_root, + source_db_name="von_db", + target_db_name="scratch", + drop_target=False, + ) + assert (backup_root / "prelude.json").read_text() == original From 9e12bc919c0d5ff57e298b0edac159aaf918a0a3 Mon Sep 17 00:00:00 2001 From: witbrock Date: Thu, 10 Sep 2026 13:45:46 -0400 Subject: [PATCH 2/2] chore(tasks): reuse merged Jira creation authority binding --- src/backend/mcp_server/mcp_stdio_server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/backend/mcp_server/mcp_stdio_server.py b/src/backend/mcp_server/mcp_stdio_server.py index c6436084f..04fbf5d0c 100644 --- a/src/backend/mcp_server/mcp_stdio_server.py +++ b/src/backend/mcp_server/mcp_stdio_server.py @@ -1046,8 +1046,6 @@ async def list_tools() -> list[Tool]: "jira_get_issue", "jira_get_myself", "jira_get_transitions", - "jira_get_project_issue_types", - "jira_create_issue", "jira_update_issue", "jira_search", "turn_execution_get",