From b285e30dff981c4c070b8c351df69e37dcd03358 Mon Sep 17 00:00:00 2001 From: witbrock Date: Thu, 10 Sep 2026 14:37:44 -0400 Subject: [PATCH] Preserve project refresh state and nested Jira configuration (JVNAUTOSCI-2737) --- ...t_jira_to_von_migration_jvnautosci_1223.md | 8 ++- .../internal_mcp/jira_migration_resources.py | 11 ++++ .../services/jira_source_retention_service.py | 48 ++++++++++++++ src/backend/services/task_project_service.py | 33 +++++++++- .../test_jira_source_retention_service.py | 62 +++++++++++++++++++ tests/backend/test_task_project_service.py | 23 ++++++- 6 files changed, 179 insertions(+), 6 deletions(-) diff --git a/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md b/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md index ad47ca429..b3cbbeab9 100644 --- a/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md +++ b/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md @@ -55,7 +55,13 @@ Never infer deletion authority from a missing source issue. `scripts/capture_jira_project_sources.py` captures project and site source configuration through named Jira MCP resources and associates board/filter -collections. Inspect its report for unsupported or inaccessible resources. +collections. It includes screen tabs and ordered fields, issue-type/screen +scheme mappings, and paginated field-configuration items. The service's +`capture_configuration_details` can supplement an existing site archive without +recapturing boards; retain the previous archive reference when associating the +supplement. Inspect the report for unsupported or inaccessible resources. +Classic configuration APIs can return empty results for team-managed projects; +that does not establish retention of their app-owned forms or configuration. Automation exports, cloud exports, app-owned data, forms and linked external documents require explicit capture or an evidence-backed scope disposition; REST access failures are not proof that a resource is empty. Keep these private diff --git a/src/backend/integrations/internal_mcp/jira_migration_resources.py b/src/backend/integrations/internal_mcp/jira_migration_resources.py index 9e93748b0..4ac1a4077 100644 --- a/src/backend/integrations/internal_mcp/jira_migration_resources.py +++ b/src/backend/integrations/internal_mcp/jira_migration_resources.py @@ -36,6 +36,17 @@ "priorities": ("api/3", "priority", False), "workflows": ("api/3", "workflow/search", True), "screens": ("api/3", "screens", True), + "screen_tabs": ("api/3", "screens/{id}/tabs", False), + "screen_tab_fields": ("api/3", "screens/{id}/tabs/{secondary}/fields", False), + "screen_schemes": ("api/3", "screenscheme", True), + "issue_type_screen_schemes": ("api/3", "issuetypescreenscheme", True), + "issue_type_screen_mappings": ("api/3", "issuetypescreenscheme/mapping", True), + "issue_type_schemes": ("api/3", "issuetypescheme", True), + "issue_type_scheme_items": ("api/3", "issuetypescheme/mapping", True), + "field_configurations": ("api/3", "fieldconfiguration", True), + "field_configuration_items": ("api/3", "fieldconfiguration/{id}/fields", True), + "field_configuration_schemes": ("api/3", "fieldconfigurationscheme", True), + "field_configuration_mappings": ("api/3", "fieldconfigurationscheme/mapping", True), "filters": ("api/3", "filter/search", True), "filter_issues": ("api/3", "search/jql", True), "filter": ("api/3", "filter/{id}", False), diff --git a/src/backend/services/jira_source_retention_service.py b/src/backend/services/jira_source_retention_service.py index d56075af8..00493e1d7 100644 --- a/src/backend/services/jira_source_retention_service.py +++ b/src/backend/services/jira_source_retention_service.py @@ -343,6 +343,49 @@ async def capture_project(proxy, project_key): } +async def capture_configuration_details(proxy, *, screens=None): + """Retain scheme mappings and nested field layouts, not only their names. + + Jira's classic configuration APIs can return an empty set for team-managed + projects. Preserve the exact responses; these reads do not certify forms or + app-owned configuration, which still need their own source dispositions. + This function also supports supplementing an existing site archive without + downloading every board and issue-membership snapshot again. + """ + resources = {} + for resource in ( + "screen_schemes", + "issue_type_screen_schemes", + "issue_type_screen_mappings", + "issue_type_schemes", + "issue_type_scheme_items", + "field_configurations", + "field_configuration_schemes", + "field_configuration_mappings", + ): + resources[resource] = await capture_resource(proxy, resource, item_key="values") + if screens is None: + resources["screens"] = await capture_resource( + proxy, "screens", item_key="values" + ) + screens = resources["screens"]["items"] + for screen in screens: + screen_id = str(screen["id"]) + tabs = await capture_resource(proxy, "screen_tabs", screen_id) + resources[f"screen_tabs:{screen_id}"] = tabs + for tab in tabs["items"]: + tab_id = str(tab["id"]) + resources[f"screen_tab_fields:{screen_id}:{tab_id}"] = ( + await capture_resource(proxy, "screen_tab_fields", screen_id, tab_id) + ) + for configuration in resources["field_configurations"]["items"]: + config_id = str(configuration["id"]) + resources[f"field_configuration_items:{config_id}"] = await capture_resource( + proxy, "field_configuration_items", config_id, item_key="values" + ) + return resources + + async def capture_site_configuration(proxy, *, site_url): """Shared definitions are captured once and referenced by project records. @@ -399,6 +442,11 @@ async def capture_site_configuration(proxy, *, site_url): item_key="values", ) ) + resources.update( + await capture_configuration_details( + proxy, screens=resources["screens"]["items"] + ) + ) for board in resources["boards"]["items"]: board_id = board["id"] for resource in ( diff --git a/src/backend/services/task_project_service.py b/src/backend/services/task_project_service.py index 4b276d8df..4d0ae6557 100644 --- a/src/backend/services/task_project_service.py +++ b/src/backend/services/task_project_service.py @@ -111,7 +111,7 @@ def ensure_jira_project( "source_id": str(project["id"]), "organisation_concept_id": organisation_concept_id, "source_url": project.get("self"), - "description": project.get("description", ""), + "description": project.get("description", old.get("description", "")), "collection_concept_ids": sorted( set(old.get("collection_concept_ids", [])) | {collection_id} ), @@ -134,7 +134,36 @@ def ensure_jira_project( if existing_project_concept_id: record["domain_project_concept_id"] = existing_project_concept_id if prior: - update_concept(project_id, {f"attributes.{PROJECT_RECORD}": record}) + # Issue responses carry a partial project object. Refresh only fields + # this source supplies, never a read/modify/write copy of the whole + # record: a concurrent archive publication or writer decision must not + # be replaced by the importer's older snapshot. + fields = { + "name", + "key", + "source_system", + "source_site", + "source_id", + "organisation_concept_id", + "source_url", + } + if "description" in project: + fields.add("description") + if collection_id not in old.get("collection_concept_ids", []): + fields.add("collection_concept_ids") + if archive_reference: + fields.add("source_archive") + if "source_archive_history" in record: + fields.add("source_archive_history") + if existing_project_concept_id: + fields.add("domain_project_concept_id") + updates = { + f"attributes.{PROJECT_RECORD}.{field}": record[field] + for field in fields + if old.get(field) != record[field] + } + if updates: + update_concept(project_id, updates) else: create_concept( name=record["name"], diff --git a/tests/backend/test_jira_source_retention_service.py b/tests/backend/test_jira_source_retention_service.py index 6ecb4a760..8232f5f85 100644 --- a/tests/backend/test_jira_source_retention_service.py +++ b/tests/backend/test_jira_source_retention_service.py @@ -151,6 +151,68 @@ def test_resource_paths_cannot_redirect_credentials(): resource_request("project", "..") +@pytest.mark.asyncio +async def test_configuration_capture_retains_nested_layout_and_all_field_pages(): + class ConfigurationSource: + async def get_migration_resource( + self, resource, identifier="", start_at=0, **kw + ): + if resource == "screen_tabs": + return [{"id": 17, "name": "Details"}] + if resource == "screen_tab_fields": + assert identifier == "12" and kw["secondary_id"] == "17" + return [{"id": "summary"}, {"id": "customfield_42"}] + if resource == "field_configurations": + return {"values": [{"id": 3}], "total": 1, "isLast": True} + if resource == "field_configuration_items": + return { + "startAt": start_at, + "total": 2, + "values": [ + {"id": f"field_{start_at}", "isRequired": bool(start_at)} + ], + } + return {"values": [], "total": 0, "isLast": True} + + result = await retention.capture_configuration_details( + ConfigurationSource(), screens=[{"id": 12, "name": "Create"}] + ) + assert [r["id"] for r in result["screen_tab_fields:12:17"]["items"]] == [ + "summary", + "customfield_42", + ] + assert result["field_configuration_items:3"]["count"] == 2 + assert result["field_configuration_items:3"]["items"][1]["isRequired"] is True + assert all(r["complete"] for r in result.values()) + + +@pytest.mark.asyncio +async def test_unavailable_screen_layout_stays_an_explicit_capture_gap(): + class UnavailableSource: + async def get_migration_resource(self, resource, **kw): + if resource == "screen_tabs": + return {"success": False, "status_code": 403, "error": "forbidden"} + return {"values": [], "total": 0, "isLast": True} + + result = await retention.capture_configuration_details( + UnavailableSource(), screens=[{"id": 12}] + ) + assert not result["screen_tabs:12"]["complete"] + assert result["screen_tabs:12"]["pages"][0]["status_code"] == 403 + assert not any(k.startswith("screen_tab_fields:") for k in result) + + +def test_nested_configuration_resources_use_fixed_read_paths(): + assert resource_request("screen_tab_fields", "12", "17") == ( + "/rest/api/3/screens/12/tabs/17/fields", + {}, + ) + assert resource_request("field_configuration_items", "3", start_at=100) == ( + "/rest/api/3/fieldconfiguration/3/fields", + {"startAt": 100, "maxResults": 100}, + ) + + def test_archive_read_requires_access_and_verifies_bytes(monkeypatch): from src.backend.services import computer_file_copy_service as files diff --git a/tests/backend/test_task_project_service.py b/tests/backend/test_task_project_service.py index 1f136b080..676280a70 100644 --- a/tests/backend/test_task_project_service.py +++ b/tests/backend/test_task_project_service.py @@ -7,6 +7,7 @@ def test_project_collection_rerun_preserves_identity_and_writer(monkeypatch): docs = {} + publish_during_refresh = False monkeypatch.setattr(projects, "can_access_concept", lambda key: True) monkeypatch.setattr( projects, @@ -23,8 +24,16 @@ def create(**kwargs): return docs[key] def update(key, fields): + if publish_during_refresh: + record = docs[key]["attributes"]["task_project"] + record.update(writer="von", site_source_archive={"content_sha256": "new"}) + record["collection_concept_ids"].append("#V#new_board_collection") for path, value in fields.items(): - docs[key]["attributes"][path.split(".", 1)[1]] = copy.deepcopy(value) + target = docs[key] + parts = path.split(".") + for part in parts[:-1]: + target = target.setdefault(part, {}) + target[parts[-1]] = copy.deepcopy(value) monkeypatch.setattr(projects, "create_concept", create) monkeypatch.setattr(projects, "update_concept", update) @@ -32,18 +41,26 @@ def update(key, fields): "id": "12", "key": "KKAT", "name": "KnowKat", + "description": "The complete project description", "self": "https://jira.example/rest/api/3/project/12", } first = projects.ensure_jira_project(source, actor_concept_id="#V#owner") project_id = first["project_concept_id"] - docs[project_id]["attributes"]["task_project"]["writer"] = "von" + publish_during_refresh = True + partial_source = { + key: value for key, value in source.items() if key != "description" + } second = projects.ensure_jira_project( - {**source, "name": "KnowKat updated"}, actor_concept_id="#V#owner" + {**partial_source, "name": "KnowKat updated"}, actor_concept_id="#V#owner" ) assert len(docs) == 2 assert second["project_concept_id"] == project_id assert second["collection"]["project_concept_ids"] == [project_id] assert second["project"]["writer"] == "von" + assert second["project"]["name"] == "KnowKat updated" + assert second["project"]["description"] == "The complete project description" + assert second["project"]["site_source_archive"] == {"content_sha256": "new"} + assert "#V#new_board_collection" in second["project"]["collection_concept_ids"] assert ( projects.validate_task_membership(project_id, second["collection_concept_ids"])[ 1