fix(harvester): map CERN/EP report numbers and match by apprn

Author: Mohammed Taha Khan

site/cds_rdm/inspire_harvester/load/matcher.py

@@ -104,11 +104,27 @@ def query(self):
 
 @dataclass(frozen=True)
 class ReportNumberMatchFilter(FilterCandidate):
-    """Match a CDS report number."""
+    """Match a CDS report number in metadata.identifiers."""
 
     @property
     def query(self):
         """Build the CDS report number query."""
+        return [
+            dsl.Q("term", **{"metadata.identifiers.scheme": "cdsrn"}),
+            dsl.Q(
+                "term",
+                **{"metadata.identifiers.identifier": self.value},
+            ),
+        ]
+
+
+@dataclass(frozen=True)
+class RelatedReportNumberMatchFilter(FilterCandidate):
+    """Match a CDS report number in metadata.related_identifiers."""
+
+    @property
+    def query(self):
+        """Build the related CDS report number query."""
         return [
             dsl.Q("term", **{"metadata.related_identifiers.scheme": "cdsrn"}),
             dsl.Q(
@@ -118,6 +134,22 @@ def query(self):
         ]
 
 
+@dataclass(frozen=True)
+class ApprovalReportNumberMatchFilter(FilterCandidate):
+    """Match an EP/approval report number (apprn) in metadata.identifiers."""
+
+    @property
+    def query(self):
+        """Build the approval report number query."""
+        return [
+            dsl.Q("term", **{"metadata.identifiers.scheme": "apprn"}),
+            dsl.Q(
+                "term",
+                **{"metadata.identifiers.identifier": self.value},
+            ),
+        ]
+
+
 class RecordMatcher:
     """Finds existing CDS records that match an incoming INSPIRE entry."""
 
@@ -163,16 +195,21 @@ def _retrieve_identifier(self, identifiers, scheme) -> Optional[str]:
     def _build_filter_priority(self, entry, inspire_id, cdsrdm_id):
         """Build the priority-ordered record match candidates."""
         doi = entry.get("pids", {}).get("doi", {}).get("identifier")
+        identifiers = entry["metadata"].get("identifiers", [])
         related_identifiers = entry["metadata"].get("related_identifiers", [])
 
         cds_id = self._retrieve_identifier(related_identifiers, "cds")
         arxiv_id = self._retrieve_identifier(related_identifiers, "arxiv")
-        report_number = self._retrieve_identifier(related_identifiers, "cdsrn")
+        report_number = self._retrieve_identifier(identifiers, "cdsrn")
+        related_report_number = self._retrieve_identifier(related_identifiers, "cdsrn")
+        approval_report_number = self._retrieve_identifier(identifiers, "apprn")
         return [
             ParentMatchFilter(value=cdsrdm_id),
             CDSIdentifierMatchFilter(value=cds_id),
             DOIMatchFilter(value=doi),
             InspireIdentifierMatchFilter(value=inspire_id),
             ArxivIdentifierMatchFilter(value=arxiv_id),
             ReportNumberMatchFilter(value=report_number),
+            RelatedReportNumberMatchFilter(value=related_report_number),
+            ApprovalReportNumberMatchFilter(value=approval_report_number),
         ]

site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py

@@ -14,9 +14,34 @@
 from idutils.normalizers import normalize_isbn
 from idutils.validators import is_doi
 
+from cds_rdm import schemes
 from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase
 
 
+def _committee_approval_prefixes():
+    """Return configured committee approval report-number prefixes."""
+    communities = current_app.config.get("CDS_COMMITTEE_APPROVAL_COMMUNITIES", {})
+    return {
+        cfg.get("report_number", {}).get("prefix")
+        for cfg in communities.values()
+        if cfg.get("report_number", {}).get("prefix")
+    }
+
+
+def _is_approval_report_number(value):
+    """Return True if value is a valid EP/approval report number.
+
+    Requires both a configured committee prefix and a value accepted by the
+    ``apprn`` scheme validator, so prefix look-alikes fall back to ``cdsrn``
+    instead of failing record validation.
+    """
+    if not value:
+        return False
+    if not schemes.is_approval_report_number(value):
+        return False
+    return any(value.startswith(prefix) for prefix in _committee_approval_prefixes())
+
+
 @dataclass(frozen=True)
 class DOIMapper(MapperBase):
     """Mapper for DOI identifiers."""
@@ -104,6 +129,19 @@ def map_value(self, src_record, ctx, logger):
                     "Unexpected schema in external_system_identifiers. "
                     f"| details: schema={schema}, value={value}"
                 )
+
+        # Report numbers on the record itself:
+        # - EP/approval numbers (configured prefixes) → apprn
+        # - other CERN- report numbers → cdsrn
+        for rn in src_metadata.get("report_numbers", []):
+            report_number = rn.get("value")
+            if not report_number:
+                continue
+            if _is_approval_report_number(report_number):
+                identifiers.append({"identifier": report_number, "scheme": "apprn"})
+            elif report_number.startswith("CERN-"):
+                identifiers.append({"identifier": report_number, "scheme": "cdsrn"})
+
         unique_ids = [dict(t) for t in {tuple(sorted(d.items())) for d in identifiers}]
         return unique_ids
 
@@ -209,12 +247,21 @@ def map_value(self, src_record, ctx, logger):
                     }
                 )
 
+            # Non-CERN- / non-approval report numbers stay related (scheme cdsrn).
+            # CERN- cdsrn and apprn values are handled by IdentifiersMapper.
             report_numbers = src_metadata.get("report_numbers", [])
             for rn in report_numbers:
+                report_number = rn.get("value")
+                if (
+                    not report_number
+                    or report_number.startswith("CERN-")
+                    or _is_approval_report_number(report_number)
+                ):
+                    continue
                 identifiers.append(
                     {
                         "scheme": "cdsrn",
-                        "identifier": f"{rn['value']}",
+                        "identifier": report_number,
                         "relation_type": {"id": "isvariantformof"},
                         "resource_type": {"id": ctx.resource_type.value},
                     }

site/tests/inspire_harvester/test_transformer.py

@@ -217,6 +217,79 @@ def test_transform_funding_missing_award_errors():
     assert "Award not found in vocabulary" in ctx.errors[0]
 
 
+def test_transform_report_numbers_as_identifiers(running_app):
+    """CERN- report numbers go to identifiers; EP prefixes use apprn."""
+    from flask import current_app
+
+    current_app.config["CDS_COMMITTEE_APPROVAL_COMMUNITIES"] = {
+        "ep-community": {
+            "report_number": {"prefix": "CERN-EP"},
+        }
+    }
+    src_metadata = {
+        "report_numbers": [
+            {"value": "CERN-EP-2026-001"},
+            {"value": "CERN-THESIS-2010-364"},
+            {"value": "DESY-24-001"},
+        ]
+    }
+    ctx = MetadataSerializationContext(
+        resource_type=ResourceType.OTHER, inspire_id="12345"
+    )
+    logger = Logger(inspire_id="12345")
+
+    identifiers = IdentifiersMapper().map_value(
+        {"metadata": src_metadata, "created": "2023-01-01"}, ctx, logger
+    )
+    related = RelatedIdentifiersMapper().map_value(
+        {"metadata": src_metadata, "created": "2023-01-01"}, ctx, logger
+    )
+
+    assert {"identifier": "CERN-EP-2026-001", "scheme": "apprn"} in identifiers
+    assert {"identifier": "CERN-THESIS-2010-364", "scheme": "cdsrn"} in identifiers
+    assert not any(i.get("scheme") == "cdsrn" and i.get("identifier") == "CERN-EP-2026-001" for i in identifiers)
+    assert any(
+        i.get("scheme") == "cdsrn" and i.get("identifier") == "DESY-24-001"
+        for i in related
+    )
+    assert not any(i.get("identifier") == "CERN-EP-2026-001" for i in related)
+    assert not any(i.get("identifier") == "CERN-THESIS-2010-364" for i in related)
+
+
+def test_matcher_includes_approval_report_number(running_app):
+    """Matcher searches by apprn as well as cdsrn."""
+    from cds_rdm.inspire_harvester.load.matcher import (
+        ApprovalReportNumberMatchFilter,
+        RecordMatcher,
+        RelatedReportNumberMatchFilter,
+        ReportNumberMatchFilter,
+    )
+
+    entry = {
+        "pids": {},
+        "metadata": {
+            "identifiers": [
+                {"scheme": "cdsrn", "identifier": "CERN-THESIS-2010-364"},
+                {"scheme": "apprn", "identifier": "CERN-EP-2026-001"},
+            ],
+            "related_identifiers": [
+                {"scheme": "cdsrn", "identifier": "DESY-24-001"},
+                {"scheme": "inspire", "identifier": "12345"},
+            ],
+        },
+    }
+    candidates = RecordMatcher()._build_filter_priority(entry, "12345", None)
+    by_type = {type(c): c for c in candidates if c.value}
+
+    assert by_type[ReportNumberMatchFilter].value == "CERN-THESIS-2010-364"
+    assert by_type[RelatedReportNumberMatchFilter].value == "DESY-24-001"
+    assert by_type[ApprovalReportNumberMatchFilter].value == "CERN-EP-2026-001"
+    assert any(
+        q.to_dict() == {"term": {"metadata.identifiers.scheme": "apprn"}}
+        for q in by_type[ApprovalReportNumberMatchFilter].query
+    )
+
+
 @patch("cds_rdm.inspire_harvester.transform.mappers.identifiers.is_doi")
 def test_transform_dois_valid_external(mock_is_doi, running_app):
     """Test DOIMapper with valid external DOI."""