feat(conformance): close 206 uncovered constraints; coverage 18.8% → 40.7%

conformance/mutation_coverage.py

@@ -136,6 +136,35 @@ def apply(schema: Dict[str, Any]) -> None:
     return apply
 
 
+def is_unfalsifiable(node: Dict[str, Any], keyword: str) -> bool:
+    """True when deleting `keyword` cannot change which documents validate.
+
+    The common case: a node carries both an ``enum`` and a narrower-looking
+    companion constraint (``pattern``, ``minLength``, ``type``, ``format``).
+    The enum already restricts the instance to a fixed set, so if every member
+    of that set satisfies the companion, the companion is dead weight -- and
+    NO test case could ever catch its removal.
+
+    Reporting that as "uncovered" would be crying wolf: it blames the corpus
+    for a gap the corpus cannot close. It is a finding about the *schema*
+    instead, so it gets its own bucket.
+    """
+    enum = node.get("enum")
+    if not isinstance(enum, list) or not enum or keyword == "enum":
+        return False
+    try:
+        from jsonschema import Draft202012Validator
+    except ImportError:  # pragma: no cover
+        return False
+    with_kw = {k: v for k, v in node.items() if k != "enum"}
+    if keyword not in with_kw:
+        return False
+    without_kw = {k: v for k, v in with_kw.items() if k != keyword}
+    a = Draft202012Validator(with_kw)
+    b = Draft202012Validator(without_kw)
+    return all(a.is_valid(v) == b.is_valid(v) for v in enum)
+
+
 def corpus_is_red(cases: List[Dict[str, Any]], schema: Dict[str, Any]) -> bool:
     """True when at least one case fails against this (mutated) schema."""
     validators = _build_validators(schema)
@@ -157,37 +186,54 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
         help="exit non-zero if coverage falls below this percentage",
     )
     ap.add_argument("--limit", type=int, default=None, help="sample N mutations")
+    ap.add_argument(
+        "--group",
+        metavar="NAME",
+        help="measure only one corpus file's cases (its stem). Answers 'does "
+        "THIS group pin these constraints', which is what you want while "
+        "authoring one group -- and it isolates you from a sibling group that "
+        "is mid-edit.",
+    )
+    ap.add_argument(
+        "--pointer-filter",
+        metavar="SUBSTR",
+        help="only report constraints whose pointer contains SUBSTR",
+    )
     args = ap.parse_args(argv)
 
     report: Dict[str, Any] = {}
     worst = 100.0
 
     for version in args.versions or discover_versions():
         try:
-            cases = load_corpus(version, include_optional=True)
+            cases = load_corpus(version, include_optional=True, group=args.group)
         except CorpusError as exc:
             print(f"mutation-coverage: {exc}", file=sys.stderr)
             return 2
         schema = load_schema(version)
 
         mutations: List[Mutation] = []
         _walk(schema, "", mutations)
+        if args.pointer_filter:
+            mutations = [m for m in mutations if args.pointer_filter in m[0]]
         if args.limit:
             mutations = mutations[: args.limit]
 
         # Sanity: the corpus must be green before anything is mutated, or every
         # mutation trivially "fails" and the whole report is meaningless.
         if corpus_is_red(cases, copy.deepcopy(schema)):
             print(
-                f"mutation-coverage: the {version} corpus is not green against the "
-                "unmutated schema; fix that first -- otherwise every constraint "
-                "reads as covered.",
+                f"mutation-coverage: the {version} corpus"
+                + (f" (group {args.group})" if args.group else "")
+                + " is not green against the unmutated schema; fix that first -- "
+                "otherwise every constraint reads as covered.",
                 file=sys.stderr,
             )
             return 2
 
         covered: List[str] = []
         uncovered: List[Tuple[str, str]] = []
+        redundant: List[Tuple[str, str]] = []
         for pointer, label, apply in mutations:
             mutated = copy.deepcopy(schema)
             try:
@@ -198,6 +244,18 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
                 continue
             if corpus_is_red(cases, mutated):
                 covered.append(f"{pointer}: {label}")
+                continue
+            # Before blaming the corpus, ask whether ANY case could have caught
+            # this. A constraint the schema makes unfalsifiable is not a
+            # coverage gap.
+            keyword = label.replace("delete ", "").split()[0]
+            node_ptr = pointer.rsplit("/", 1)[0]
+            try:
+                node = _resolve(schema, node_ptr)
+            except (KeyError, IndexError, TypeError):
+                node = None
+            if isinstance(node, dict) and is_unfalsifiable(node, keyword):
+                redundant.append((pointer, label))
             else:
                 uncovered.append((pointer, label))
 
@@ -209,13 +267,21 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
             "mutations": total,
             "covered": len(covered),
             "uncovered": [{"pointer": p, "mutation": m} for p, m in uncovered],
+            "redundant": [{"pointer": p, "mutation": m} for p, m in redundant],
             "coveragePct": round(pct, 1),
         }
 
         if args.format == "text":
-            print(f"\n{version}: {len(cases)} cases vs {total} schema constraints")
+            print(f"\n{version}: {len(cases)} cases vs {total} falsifiable constraints")
             print(f"  covered   {len(covered):4}  ({pct:.1f}%)")
             print(f"  UNCOVERED {len(uncovered):4}")
+            if redundant:
+                print(
+                    f"  redundant {len(redundant):4}  (schema findings, not corpus "
+                    "gaps: an enum already makes these unfalsifiable)"
+                )
+                for pointer, label in redundant[:10]:
+                    print(f"    {pointer}  --  {label}")
             for pointer, label in uncovered[:40]:
                 print(f"    {pointer}  --  {label}")
             if len(uncovered) > 40: