DOOP: add twelve pinned datasets and portable preparation

docs/benchmarks.md

@@ -58,6 +58,66 @@ Each invocation prints one line per phase (DSL build → emit →
 compile → load → run) with wall-clock timings — useful when
 diagnosing where time is going on your box.
 
+## Twelve-dataset DOOP corpus
+
+`examples/doop_benchmark.py` prepares twelve real DaCapo
+23.11-MR2-chopin applications from the
+[published FlowLog facts](https://huggingface.co/datasets/NemoYuu/flowlog_benchmark/tree/main/dataset/csv).
+`examples/doop_suite/datasets.json` pins the corpus revision, archive SHA-256,
+archive size, and the source of the upstream reference cardinalities.
+These are fresh Chopin datasets, not aliases for the older five local datasets.
+
+The following **local scheduling tiers** use upstream reference `VarPointsTo`
+cardinality, not input size or measured SRDatalog results. They are not official
+DOOP dataset editions. H2O, for example, has substantially more raw input than
+Jython but a much smaller upstream points-to result.
+
+| Tier | Reference VPT rows | Applications |
+|---|---:|---|
+| small | < 15 million | xalan, zxing, biojava, pmd, sunflow |
+| medium | 15–<30 million | h2o, spring |
+| large | 30–<100 million | batik, eclipse, fop, h2 |
+| xlarge | >= 100 million | jython |
+
+```bash
+python examples/doop_benchmark.py list
+python examples/doop_benchmark.py fetch --all --root /path/to/doop-data
+python examples/doop_benchmark.py prepare --all --root /path/to/doop-data
+
+# Select named applications or a workload tier instead:
+python examples/doop_benchmark.py prepare --dataset xalan jython \
+    --root /path/to/doop-data
+python examples/doop_benchmark.py prepare --tier medium \
+    --root /path/to/doop-data
+```
+
+Python 3.10+ and the external `sort` command are required for preparation.
+Downloading all archives requires approximately 1.74 GB; the complete raw facts
+require approximately 28.3 GB before normalized inputs, dictionaries, build
+caches or result exports. Keep all data outside the source checkout.
+`--archive-cache DIR` optionally reuses a read-only archive cache after checksum
+verification. `DOOP_SORT_TMPDIR` can select an existing scratch directory.
+
+Preparation preserves the complete `MainClass` set and uses one shared symbol
+dictionary per dataset. It derives all 39 declared integer TSV input files,
+including descriptors and heap types, then materializes each projected relation
+as a set with `sort -u`; it never samples rows or adds synthetic roots.
+Required files, arities, signed-int32 numeric domains, and functional attributes
+needed by the normalization are checked explicitly.
+The instantiated program currently uses 37 of those inputs and has 37 derived
+relations; `Var_DeclaringMethod` and `isVirtualMethodInvocation_Insn` are declared
+but unused. Preparation reports all declared files; execution reports the
+actually consumed input rows and bytes separately.
+
+Each `prepared/APP/` contains the input CSV files (tab-delimited despite their
+extension), `meta.json`, `str2num.json`, and `manifest.json`. The manifest records
+raw/prepared hashes, prepared rows and bytes, entrypoints, and provenance.
+The status `prepared_not_engine_validated` deliberately does not claim query
+correctness. An incomplete preparation is not published; existing prepared
+datasets are verified rather than overwritten. Do not reuse another dataset's
+interned constants or compare old-generation facts under the same application
+name without recording that change.
+
 ## Regenerating from Nim
 
 When upstream Nim sources change, regenerate every benchmark with:

examples/doop_benchmark.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""Prepare and inspect the pinned twelve-application DOOP benchmark suite."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from doop_suite.catalog import (
+  extract_archive,
+  fetch_archive,
+  load_catalog,
+  select_datasets,
+  verify_prepared,
+  verify_raw,
+)
+
+
+def selection(parser: argparse.ArgumentParser, *, required: bool = True) -> None:
+  group = parser.add_mutually_exclusive_group(required=required)
+  group.add_argument('--all', action='store_true', help='Select all twelve datasets')
+  group.add_argument('--dataset', nargs='+', metavar='NAME', help='Select named datasets')
+  group.add_argument('--tier', choices=list(load_catalog()['tiers']))
+
+
+def prepare_dataset(dataset: dict, root: Path, archive_cache: Path | None = None) -> dict:
+  from doop_suite.prepare import prepare
+
+  output = root / 'prepared' / dataset['name']
+  if output.exists():
+    return verify_prepared(output, dataset)
+  archive = fetch_archive(dataset, root, archive_cache)
+  raw = root / 'raw' / dataset['name']
+  if raw.exists():
+    verify_raw(archive, dataset, raw)
+  else:
+    raw = extract_archive(archive, dataset, root)
+  catalog = load_catalog()
+  provenance = dict(
+    dataset,
+    corpus=catalog['corpus'],
+    revision=catalog['revision'],
+    repository=catalog['repository'],
+    reference_source=catalog['reference_source'],
+  )
+  return prepare(raw, output, provenance=provenance)
+
+
+def main(argv: list[str] | None = None) -> int:
+  parser = argparse.ArgumentParser(description=__doc__)
+  commands = parser.add_subparsers(dest='command', required=True)
+  listing = commands.add_parser('list', help='Show tiers, names and upstream reference sizes')
+  selection(listing, required=False)
+  listing.add_argument('--json', action='store_true')
+  for command in ('fetch', 'prepare'):
+    action = commands.add_parser(
+      command,
+      help='Download pinned archives'
+      if command == 'fetch'
+      else 'Download and normalize complete input relation sets',
+    )
+    selection(action)
+    action.add_argument('--root', type=Path, required=True, help='External data/artifact directory')
+    action.add_argument('--archive-cache', type=Path, help='Optional read-only existing zip cache')
+  args = parser.parse_args(argv)
+  try:
+    datasets = select_datasets(args.dataset, args.tier)
+    if args.command == 'list':
+      if args.json:
+        print(json.dumps(datasets, indent=2))
+      else:
+        print(
+          'Local tiers use upstream reference VarPointsTo rows, NOT input size or local results.'
+        )
+        print(f'{"DATASET":12} {"TIER":8} {"REFERENCE VPT":>15} {"ARCHIVE MB":>12}')
+        for item in datasets:
+          print(
+            f'{item["name"]:12} {item["tier"]:8} {item["reference_vpt_rows"]:>15,}'
+            f' {item["archive_bytes"] / 1_000_000:>12.1f}'
+          )
+      return 0
+    root = args.root.resolve()
+    for dataset in datasets:
+      print(f'[{args.command}] {dataset["name"]}', flush=True)
+      if args.command == 'fetch':
+        archive = fetch_archive(dataset, root, args.archive_cache)
+        print(
+          json.dumps({'dataset': dataset['name'], 'archive': str(archive), 'verified': True}),
+          flush=True,
+        )
+      else:
+        manifest = prepare_dataset(dataset, root, args.archive_cache)
+        print(
+          json.dumps(
+            {
+              'dataset': dataset['name'],
+              'status': manifest['status'],
+              'prepared_input_rows': sum(r['rows'] for r in manifest['relations'].values()),
+              'prepared_input_bytes': sum(r['bytes'] for r in manifest['relations'].values()),
+              'directory': str(root / 'prepared' / dataset['name']),
+            }
+          ),
+          flush=True,
+        )
+    return 0
+  except (OSError, ValueError, RuntimeError) as error:
+    print(f'[error] {error}', file=sys.stderr)
+    return 1
+
+
+if __name__ == '__main__':
+  raise SystemExit(main())

examples/doop_suite/__init__.py

@@ -0,0 +1 @@
+"""Reproducible DOOP datasets and opt-in benchmark execution."""

examples/doop_suite/catalog.py

@@ -0,0 +1,155 @@
+"""Pinned public DOOP facts; dataset identity is independent of workload tier."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import stat
+import tempfile
+import urllib.request
+import zipfile
+import zlib
+from pathlib import Path, PurePosixPath
+
+CATALOG = Path(__file__).with_name('datasets.json')
+
+
+def digest(path: Path) -> str:
+  result = hashlib.sha256()
+  with path.open('rb') as stream:
+    for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b''):
+      result.update(chunk)
+  return result.hexdigest()
+
+
+def load_catalog() -> dict:
+  return json.loads(CATALOG.read_text())
+
+
+def select_datasets(names: list[str] | None = None, tier: str | None = None) -> list[dict]:
+  catalog = load_catalog()
+  datasets = catalog['datasets']
+  known = {row['name'] for row in datasets}
+  if names and (unknown := set(names) - known):
+    raise ValueError(f'Unknown DOOP datasets: {sorted(unknown)}')
+  if tier is not None and tier not in catalog['tiers']:
+    raise ValueError(f'Unknown DOOP tier: {tier}')
+  selected = [
+    row
+    for row in datasets
+    if (not names or row['name'] in names) and (tier is None or row['tier'] == tier)
+  ]
+  if not selected:
+    raise ValueError('Selection contains no datasets')
+  return selected
+
+
+def check_archive(path: Path, dataset: dict) -> None:
+  if path.stat().st_size != dataset['archive_bytes']:
+    raise ValueError(f'Archive size mismatch: {path}')
+  if digest(path) != dataset['archive_sha256']:
+    raise ValueError(f'Archive SHA256 mismatch: {path}')
+
+
+def fetch_archive(dataset: dict, root: Path, archive_cache: Path | None = None) -> Path:
+  """Reuse only byte-verified archives; never modify a supplied external cache."""
+  root = root.resolve()
+  name = dataset['name'] + '.zip'
+  if archive_cache is not None and (cached := archive_cache / name).is_file():
+    check_archive(cached, dataset)
+    return cached.resolve()
+  downloads = root / 'downloads'
+  downloads.mkdir(parents=True, exist_ok=True)
+  target = downloads / name
+  if target.exists():
+    check_archive(target, dataset)
+    return target
+  with tempfile.TemporaryDirectory(prefix=f'.{dataset["name"]}-', dir=downloads) as temporary:
+    partial = Path(temporary) / name
+    request = urllib.request.Request(dataset['url'], headers={'User-Agent': 'srdatalog-doop-suite'})
+    with urllib.request.urlopen(request, timeout=120) as response, partial.open('xb') as stream:
+      shutil.copyfileobj(response, stream, length=8 * 1024 * 1024)
+    check_archive(partial, dataset)
+    partial.rename(target)
+  return target
+
+
+def verify_raw(archive: Path, dataset: dict, directory: Path) -> None:
+  """Check a reused extraction against the pinned archive's member CRCs."""
+  check_archive(archive, dataset)
+  expected = set()
+  with zipfile.ZipFile(archive) as source:
+    for entry in source.infolist():
+      if entry.is_dir():
+        continue
+      name = PurePosixPath(entry.filename)
+      if len(name.parts) != 2 or name.parts[0] != dataset['name'] or name.suffix != '.facts':
+        raise ValueError(f'Unexpected DOOP archive member: {entry.filename}')
+      path = directory / name.name
+      expected.add(name.name)
+      if path.is_symlink() or path.stat().st_size != entry.file_size:
+        raise ValueError(f'Extracted fact size/type mismatch: {path}')
+      crc = 0
+      with path.open('rb') as stream:
+        for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b''):
+          crc = zlib.crc32(chunk, crc)
+      if crc != entry.CRC:
+        raise ValueError(f'Extracted facts changed: {path}')
+  if {path.name for path in directory.iterdir()} != expected:
+    raise ValueError(f'Unexpected or missing extracted facts in {directory}')
+
+
+def extract_archive(archive: Path, dataset: dict, root: Path) -> Path:
+  """Extract a verified archive into a new directory, rejecting links and traversal."""
+  check_archive(archive, dataset)
+  raw = root.resolve() / 'raw'
+  raw.mkdir(parents=True, exist_ok=True)
+  target = raw / dataset['name']
+  if target.exists():
+    raise FileExistsError(f'Refusing to overwrite extracted facts: {target}')
+  with tempfile.TemporaryDirectory(prefix=f'.{dataset["name"]}-', dir=raw) as temporary:
+    staging = Path(temporary)
+    with zipfile.ZipFile(archive) as source:
+      seen = set()
+      for entry in source.infolist():
+        name = PurePosixPath(entry.filename)
+        mode = entry.external_attr >> 16
+        if (
+          name.is_absolute()
+          or '..' in name.parts
+          or '\\' in entry.filename
+          or not name.parts
+          or name.parts[0] != dataset['name']
+          or stat.S_ISLNK(mode)
+          or entry.filename in seen
+        ):
+          raise ValueError(f'Unsafe or duplicate archive member: {entry.filename}')
+        seen.add(entry.filename)
+        destination = staging.joinpath(*name.parts)
+        if entry.is_dir():
+          destination.mkdir(parents=True, exist_ok=True)
+        else:
+          if len(name.parts) != 2 or name.suffix != '.facts':
+            raise ValueError(f'Unexpected DOOP archive member: {entry.filename}')
+          destination.parent.mkdir(parents=True, exist_ok=True)
+          with source.open(entry) as stream, destination.open('xb') as output:
+            shutil.copyfileobj(stream, output, length=8 * 1024 * 1024)
+      if not (staging / dataset['name'] / 'MainClass.facts').is_file():
+        raise ValueError('Archive has no MainClass.facts')
+    (staging / dataset['name']).rename(target)
+  return target
+
+
+def verify_prepared(directory: Path, dataset: dict) -> dict:
+  """Reused prepared inputs must retain their exact provenance, metadata and tuples."""
+  manifest = json.loads((directory / 'manifest.json').read_text())
+  if manifest.get('provenance', {}).get('archive_sha256') != dataset['archive_sha256']:
+    raise ValueError(f'Prepared dataset has different archive provenance: {directory}')
+  if json.loads((directory / 'meta.json').read_text()) != manifest['meta']:
+    raise ValueError(f'Prepared metadata changed: {directory}')
+  for name, info in manifest['relations'].items():
+    path = directory / (name + '.csv')
+    if path.stat().st_size != info['bytes'] or digest(path) != info['sha256']:
+      raise ValueError(f'Prepared input changed: {path}')
+  return manifest