Preserve project metadata during refresh and retain nested Jira configuration (JVNAUTOSCI-2737)

docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md

@@ -55,7 +55,13 @@ Never infer deletion authority from a missing source issue.
 
 `scripts/capture_jira_project_sources.py` captures project and site source
 configuration through named Jira MCP resources and associates board/filter
-collections. Inspect its report for unsupported or inaccessible resources.
+collections. It includes screen tabs and ordered fields, issue-type/screen
+scheme mappings, and paginated field-configuration items. The service's
+`capture_configuration_details` can supplement an existing site archive without
+recapturing boards; retain the previous archive reference when associating the
+supplement. Inspect the report for unsupported or inaccessible resources.
+Classic configuration APIs can return empty results for team-managed projects;
+that does not establish retention of their app-owned forms or configuration.
 Automation exports, cloud exports, app-owned data, forms and linked external
 documents require explicit capture or an evidence-backed scope disposition;
 REST access failures are not proof that a resource is empty. Keep these private

src/backend/integrations/internal_mcp/jira_migration_resources.py

@@ -36,6 +36,17 @@
     "priorities": ("api/3", "priority", False),
     "workflows": ("api/3", "workflow/search", True),
     "screens": ("api/3", "screens", True),
+    "screen_tabs": ("api/3", "screens/{id}/tabs", False),
+    "screen_tab_fields": ("api/3", "screens/{id}/tabs/{secondary}/fields", False),
+    "screen_schemes": ("api/3", "screenscheme", True),
+    "issue_type_screen_schemes": ("api/3", "issuetypescreenscheme", True),
+    "issue_type_screen_mappings": ("api/3", "issuetypescreenscheme/mapping", True),
+    "issue_type_schemes": ("api/3", "issuetypescheme", True),
+    "issue_type_scheme_items": ("api/3", "issuetypescheme/mapping", True),
+    "field_configurations": ("api/3", "fieldconfiguration", True),
+    "field_configuration_items": ("api/3", "fieldconfiguration/{id}/fields", True),
+    "field_configuration_schemes": ("api/3", "fieldconfigurationscheme", True),
+    "field_configuration_mappings": ("api/3", "fieldconfigurationscheme/mapping", True),
     "filters": ("api/3", "filter/search", True),
     "filter_issues": ("api/3", "search/jql", True),
     "filter": ("api/3", "filter/{id}", False),

src/backend/services/jira_source_retention_service.py

@@ -343,6 +343,49 @@ async def capture_project(proxy, project_key):
     }
 
 
+async def capture_configuration_details(proxy, *, screens=None):
+    """Retain scheme mappings and nested field layouts, not only their names.
+
+    Jira's classic configuration APIs can return an empty set for team-managed
+    projects. Preserve the exact responses; these reads do not certify forms or
+    app-owned configuration, which still need their own source dispositions.
+    This function also supports supplementing an existing site archive without
+    downloading every board and issue-membership snapshot again.
+    """
+    resources = {}
+    for resource in (
+        "screen_schemes",
+        "issue_type_screen_schemes",
+        "issue_type_screen_mappings",
+        "issue_type_schemes",
+        "issue_type_scheme_items",
+        "field_configurations",
+        "field_configuration_schemes",
+        "field_configuration_mappings",
+    ):
+        resources[resource] = await capture_resource(proxy, resource, item_key="values")
+    if screens is None:
+        resources["screens"] = await capture_resource(
+            proxy, "screens", item_key="values"
+        )
+        screens = resources["screens"]["items"]
+    for screen in screens:
+        screen_id = str(screen["id"])
+        tabs = await capture_resource(proxy, "screen_tabs", screen_id)
+        resources[f"screen_tabs:{screen_id}"] = tabs
+        for tab in tabs["items"]:
+            tab_id = str(tab["id"])
+            resources[f"screen_tab_fields:{screen_id}:{tab_id}"] = (
+                await capture_resource(proxy, "screen_tab_fields", screen_id, tab_id)
+            )
+    for configuration in resources["field_configurations"]["items"]:
+        config_id = str(configuration["id"])
+        resources[f"field_configuration_items:{config_id}"] = await capture_resource(
+            proxy, "field_configuration_items", config_id, item_key="values"
+        )
+    return resources
+
+
 async def capture_site_configuration(proxy, *, site_url):
     """Shared definitions are captured once and referenced by project records.
 
@@ -399,6 +442,11 @@ async def capture_site_configuration(proxy, *, site_url):
                         item_key="values",
                     )
                 )
+    resources.update(
+        await capture_configuration_details(
+            proxy, screens=resources["screens"]["items"]
+        )
+    )
     for board in resources["boards"]["items"]:
         board_id = board["id"]
         for resource in (

src/backend/services/task_project_service.py

@@ -111,7 +111,7 @@ def ensure_jira_project(
         "source_id": str(project["id"]),
         "organisation_concept_id": organisation_concept_id,
         "source_url": project.get("self"),
-        "description": project.get("description", ""),
+        "description": project.get("description", old.get("description", "")),
         "collection_concept_ids": sorted(
             set(old.get("collection_concept_ids", [])) | {collection_id}
         ),
@@ -134,7 +134,36 @@ def ensure_jira_project(
     if existing_project_concept_id:
         record["domain_project_concept_id"] = existing_project_concept_id
     if prior:
-        update_concept(project_id, {f"attributes.{PROJECT_RECORD}": record})
+        # Issue responses carry a partial project object. Refresh only fields
+        # this source supplies, never a read/modify/write copy of the whole
+        # record: a concurrent archive publication or writer decision must not
+        # be replaced by the importer's older snapshot.
+        fields = {
+            "name",
+            "key",
+            "source_system",
+            "source_site",
+            "source_id",
+            "organisation_concept_id",
+            "source_url",
+        }
+        if "description" in project:
+            fields.add("description")
+        if collection_id not in old.get("collection_concept_ids", []):
+            fields.add("collection_concept_ids")
+        if archive_reference:
+            fields.add("source_archive")
+            if "source_archive_history" in record:
+                fields.add("source_archive_history")
+        if existing_project_concept_id:
+            fields.add("domain_project_concept_id")
+        updates = {
+            f"attributes.{PROJECT_RECORD}.{field}": record[field]
+            for field in fields
+            if old.get(field) != record[field]
+        }
+        if updates:
+            update_concept(project_id, updates)
     else:
         create_concept(
             name=record["name"],

tests/backend/test_jira_source_retention_service.py

@@ -151,6 +151,68 @@ def test_resource_paths_cannot_redirect_credentials():
         resource_request("project", "..")
 
 
+@pytest.mark.asyncio
+async def test_configuration_capture_retains_nested_layout_and_all_field_pages():
+    class ConfigurationSource:
+        async def get_migration_resource(
+            self, resource, identifier="", start_at=0, **kw
+        ):
+            if resource == "screen_tabs":
+                return [{"id": 17, "name": "Details"}]
+            if resource == "screen_tab_fields":
+                assert identifier == "12" and kw["secondary_id"] == "17"
+                return [{"id": "summary"}, {"id": "customfield_42"}]
+            if resource == "field_configurations":
+                return {"values": [{"id": 3}], "total": 1, "isLast": True}
+            if resource == "field_configuration_items":
+                return {
+                    "startAt": start_at,
+                    "total": 2,
+                    "values": [
+                        {"id": f"field_{start_at}", "isRequired": bool(start_at)}
+                    ],
+                }
+            return {"values": [], "total": 0, "isLast": True}
+
+    result = await retention.capture_configuration_details(
+        ConfigurationSource(), screens=[{"id": 12, "name": "Create"}]
+    )
+    assert [r["id"] for r in result["screen_tab_fields:12:17"]["items"]] == [
+        "summary",
+        "customfield_42",
+    ]
+    assert result["field_configuration_items:3"]["count"] == 2
+    assert result["field_configuration_items:3"]["items"][1]["isRequired"] is True
+    assert all(r["complete"] for r in result.values())
+
+
+@pytest.mark.asyncio
+async def test_unavailable_screen_layout_stays_an_explicit_capture_gap():
+    class UnavailableSource:
+        async def get_migration_resource(self, resource, **kw):
+            if resource == "screen_tabs":
+                return {"success": False, "status_code": 403, "error": "forbidden"}
+            return {"values": [], "total": 0, "isLast": True}
+
+    result = await retention.capture_configuration_details(
+        UnavailableSource(), screens=[{"id": 12}]
+    )
+    assert not result["screen_tabs:12"]["complete"]
+    assert result["screen_tabs:12"]["pages"][0]["status_code"] == 403
+    assert not any(k.startswith("screen_tab_fields:") for k in result)
+
+
+def test_nested_configuration_resources_use_fixed_read_paths():
+    assert resource_request("screen_tab_fields", "12", "17") == (
+        "/rest/api/3/screens/12/tabs/17/fields",
+        {},
+    )
+    assert resource_request("field_configuration_items", "3", start_at=100) == (
+        "/rest/api/3/fieldconfiguration/3/fields",
+        {"startAt": 100, "maxResults": 100},
+    )
+
+
 def test_archive_read_requires_access_and_verifies_bytes(monkeypatch):
     from src.backend.services import computer_file_copy_service as files
 

tests/backend/test_task_project_service.py

@@ -7,6 +7,7 @@
 
 def test_project_collection_rerun_preserves_identity_and_writer(monkeypatch):
     docs = {}
+    publish_during_refresh = False
     monkeypatch.setattr(projects, "can_access_concept", lambda key: True)
     monkeypatch.setattr(
         projects,
@@ -23,27 +24,43 @@ def create(**kwargs):
         return docs[key]
 
     def update(key, fields):
+        if publish_during_refresh:
+            record = docs[key]["attributes"]["task_project"]
+            record.update(writer="von", site_source_archive={"content_sha256": "new"})
+            record["collection_concept_ids"].append("#V#new_board_collection")
         for path, value in fields.items():
-            docs[key]["attributes"][path.split(".", 1)[1]] = copy.deepcopy(value)
+            target = docs[key]
+            parts = path.split(".")
+            for part in parts[:-1]:
+                target = target.setdefault(part, {})
+            target[parts[-1]] = copy.deepcopy(value)
 
     monkeypatch.setattr(projects, "create_concept", create)
     monkeypatch.setattr(projects, "update_concept", update)
     source = {
         "id": "12",
         "key": "KKAT",
         "name": "KnowKat",
+        "description": "The complete project description",
         "self": "https://jira.example/rest/api/3/project/12",
     }
     first = projects.ensure_jira_project(source, actor_concept_id="#V#owner")
     project_id = first["project_concept_id"]
-    docs[project_id]["attributes"]["task_project"]["writer"] = "von"
+    publish_during_refresh = True
+    partial_source = {
+        key: value for key, value in source.items() if key != "description"
+    }
     second = projects.ensure_jira_project(
-        {**source, "name": "KnowKat updated"}, actor_concept_id="#V#owner"
+        {**partial_source, "name": "KnowKat updated"}, actor_concept_id="#V#owner"
     )
     assert len(docs) == 2
     assert second["project_concept_id"] == project_id
     assert second["collection"]["project_concept_ids"] == [project_id]
     assert second["project"]["writer"] == "von"
+    assert second["project"]["name"] == "KnowKat updated"
+    assert second["project"]["description"] == "The complete project description"
+    assert second["project"]["site_source_archive"] == {"content_sha256": "new"}
+    assert "#V#new_board_collection" in second["project"]["collection_concept_ids"]
     assert (
         projects.validate_task_membership(project_id, second["collection_concept_ids"])[
             1