From 5b2246563ed776fc16ec1e8f0cbba458fb2a24e3 Mon Sep 17 00:00:00 2001 From: nataliakokoromyti Date: Mon, 13 Jul 2026 19:26:59 -0700 Subject: [PATCH 1/2] Harden eigh benchmark against content memoization --- problems/linalg/eigh_py/eval.py | 50 +++++++++--- problems/linalg/eigh_py/test_eval.py | 115 +++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 problems/linalg/eigh_py/test_eval.py diff --git a/problems/linalg/eigh_py/eval.py b/problems/linalg/eigh_py/eval.py index 49004c70..12ba42e2 100644 --- a/problems/linalg/eigh_py/eval.py +++ b/problems/linalg/eigh_py/eval.py @@ -5,7 +5,9 @@ import re import sys import time +from hashlib import blake2b from pathlib import Path +from secrets import token_bytes as secure_token_bytes from typing import Any, Optional import torch @@ -22,6 +24,8 @@ MAX_ITERATIONS_PER_BENCHMARK = 50 BENCHMARK_INPUT_BYTES_TARGET = 256 * 1024 * 1024 +BENCHMARK_NONCE_BYTES = 16 +TORCH_SEED_MASK = (1 << 63) - 1 class PopcornOutput: @@ -155,12 +159,28 @@ def run_testing(logger: PopcornOutput, pool: multiprocessing.Pool, tests: list[T return 0 if passed else 112 -def _make_data_batch(test: TestCase, count: int): - args = dict(test.args) +def _benchmark_seed( + base_seed: int, + nonce: bytes, + generation: int, + index: int, +) -> int: + payload = f"{base_seed}:{generation}:{index}".encode("ascii") + digest = blake2b( + payload, + digest_size=8, + key=nonce, + person=b"eigh-eval-v1", + ).digest() + return int.from_bytes(digest, "little") & TORCH_SEED_MASK + + +def _make_data_batch(test: TestCase, count: int, nonce: bytes, generation: int): + base_seed = int(test.args.get("seed", 0)) data_list = [] - for _ in range(count): - if "seed" in args: - args["seed"] += 42 + for index in range(count): + args = dict(test.args) + args["seed"] = _benchmark_seed(base_seed, nonce, generation, index) data_list.append(generate_input(**args)) return data_list @@ -180,20 +200,31 @@ def _run_single_benchmark( max_repeats: int, max_time_ns: float, ) -> Stats | Any: + # This nonce separates warmup and measured inputs across every invocation, + # including leaderboard prewarm and the later scored pass. + nonce = secure_token_bytes(BENCHMARK_NONCE_BYTES) from submission import custom_kernel - data_list = _make_data_batch(test, _benchmark_batch_count(test)) - check_copy = _clone_data(data_list) + batch_count = _benchmark_batch_count(test) + warmup_data = _make_data_batch(test, batch_count, nonce, generation=0) + warmup_reference = _clone_data(warmup_data) - outputs = [custom_kernel(_clone_data(data)) for data in data_list] - for reference_data, output in zip(check_copy, outputs): + outputs = [custom_kernel(_clone_data(data)) for data in warmup_data] + for reference_data, output in zip(warmup_reference, outputs): good, message = check_implementation(reference_data, output) if not good: return message + del warmup_data, warmup_reference, outputs durations = [] bm_start_time = time.perf_counter_ns() for i in range(max_repeats): + # Cloning a warmup tensor changes its identity but not its contents. + # Generate unseen contents for every timed invocation so neither + # pointer-key nor byte-equality memoization can hit inside the timer. + data_list = _make_data_batch(test, batch_count, nonce, generation=i + 1) + check_copy = _clone_data(data_list) if recheck else None + torch.cuda.synchronize() clear_l2_cache() start_event = torch.cuda.Event(enable_timing=True) @@ -209,6 +240,7 @@ def _run_single_benchmark( good, message = check_implementation(reference_data, output) if not good: return message + del data_list, check_copy, outputs total_bm_duration = time.perf_counter_ns() - bm_start_time if i > 1 and total_bm_duration > 1e8: diff --git a/problems/linalg/eigh_py/test_eval.py b/problems/linalg/eigh_py/test_eval.py new file mode 100644 index 00000000..773e5218 --- /dev/null +++ b/problems/linalg/eigh_py/test_eval.py @@ -0,0 +1,115 @@ +import sys +import types +import unittest +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import patch + +import torch + + +def _load_eval_module(): + reference = types.ModuleType("reference") + reference.check_implementation = lambda data, output: (True, "") + reference.generate_input = lambda **kwargs: kwargs + + task = types.ModuleType("task") + task.TestSpec = dict + + utils = types.ModuleType("utils") + utils.clear_l2_cache = lambda: None + utils.set_seed = lambda seed: None + + name = "eigh_eval_under_test" + path = Path(__file__).with_name("eval.py") + source = "from __future__ import annotations\n" + path.read_text() + module = types.ModuleType(name) + module.__file__ = str(path) + with patch.dict( + sys.modules, + {name: module, "reference": reference, "task": task, "utils": utils}, + ): + exec(compile(source, str(path), "exec"), module.__dict__) + return module + + +class _FakeEvent: + def __init__(self, enable_timing=False): + self.enable_timing = enable_timing + + def record(self): + pass + + def elapsed_time(self, other): + return 1.0 + + +class FreshBenchmarkInputTest(unittest.TestCase): + def test_content_memoization_never_hits_warmup_or_previous_runs(self): + evaluator = _load_eval_module() + generated_seeds = [] + cached_outputs = {} + cache_hits = 0 + + def generate_input(**args): + generated_seeds.append(args["seed"]) + return torch.tensor([args["seed"]], dtype=torch.int64) + + def custom_kernel(data): + nonlocal cache_hits + key = int(data.item()) + if key in cached_outputs: + cache_hits += 1 + return cached_outputs[key].clone() + output = data.clone() + cached_outputs[key] = output.clone() + return output + + def check_implementation(data, output): + return torch.equal(data, output), "memoized output used for different input" + + submission = types.ModuleType("submission") + submission.custom_kernel = custom_kernel + test = evaluator.TestCase( + args={"batch": 1, "n": 1, "cond": 0, "seed": 1234}, + spec="unit-test", + ) + + with ExitStack() as stack: + stack.enter_context(patch.dict(sys.modules, {"submission": submission})) + stack.enter_context(patch.object(evaluator, "generate_input", generate_input)) + stack.enter_context( + patch.object(evaluator, "check_implementation", check_implementation) + ) + stack.enter_context( + patch.object(evaluator, "_benchmark_batch_count", return_value=1) + ) + stack.enter_context(patch.object(evaluator, "clear_l2_cache", return_value=None)) + stack.enter_context( + patch.object(evaluator.torch.cuda, "synchronize", return_value=None) + ) + stack.enter_context(patch.object(evaluator.torch.cuda, "Event", _FakeEvent)) + stack.enter_context( + patch.object(evaluator.time, "perf_counter_ns", return_value=0) + ) + stack.enter_context( + patch.object( + evaluator, + "secure_token_bytes", + side_effect=(bytes([1]) * 16, bytes([2]) * 16), + ) + ) + first = evaluator._run_single_benchmark(test, True, 4, float("inf")) + second = evaluator._run_single_benchmark(test, True, 4, float("inf")) + + self.assertIsInstance(first, evaluator.Stats) + self.assertIsInstance(second, evaluator.Stats) + self.assertEqual(first.runs, 4) + self.assertEqual(second.runs, 4) + self.assertEqual(cache_hits, 0) + self.assertEqual(len(generated_seeds), 10) + self.assertEqual(len(set(generated_seeds)), len(generated_seeds)) + + +if __name__ == "__main__": + unittest.main() From aa95bc3b986d3b25ecbddd1e976d92145e47777f Mon Sep 17 00:00:00 2001 From: nataliakokoromyti Date: Mon, 13 Jul 2026 20:07:05 -0700 Subject: [PATCH 2/2] Bound fresh-input benchmark repeats --- problems/linalg/eigh_py/eval.py | 4 +++- problems/linalg/eigh_py/test_eval.py | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/problems/linalg/eigh_py/eval.py b/problems/linalg/eigh_py/eval.py index 12ba42e2..fe2b99f9 100644 --- a/problems/linalg/eigh_py/eval.py +++ b/problems/linalg/eigh_py/eval.py @@ -23,6 +23,7 @@ MAX_ITERATIONS_PER_BENCHMARK = 50 +MAX_FRESH_REPEATS_PER_BENCHMARK = 50 BENCHMARK_INPUT_BYTES_TARGET = 256 * 1024 * 1024 BENCHMARK_NONCE_BYTES = 16 TORCH_SEED_MASK = (1 << 63) - 1 @@ -218,7 +219,8 @@ def _run_single_benchmark( durations = [] bm_start_time = time.perf_counter_ns() - for i in range(max_repeats): + repeat_limit = min(max_repeats, MAX_FRESH_REPEATS_PER_BENCHMARK) + for i in range(repeat_limit): # Cloning a warmup tensor changes its identity but not its contents. # Generate unseen contents for every timed invocation so neither # pointer-key nor byte-equality memoization can hit inside the timer. diff --git a/problems/linalg/eigh_py/test_eval.py b/problems/linalg/eigh_py/test_eval.py index 773e5218..9e452790 100644 --- a/problems/linalg/eigh_py/test_eval.py +++ b/problems/linalg/eigh_py/test_eval.py @@ -99,15 +99,15 @@ def check_implementation(data, output): side_effect=(bytes([1]) * 16, bytes([2]) * 16), ) ) - first = evaluator._run_single_benchmark(test, True, 4, float("inf")) - second = evaluator._run_single_benchmark(test, True, 4, float("inf")) + first = evaluator._run_single_benchmark(test, True, 60, float("inf")) + second = evaluator._run_single_benchmark(test, True, 60, float("inf")) self.assertIsInstance(first, evaluator.Stats) self.assertIsInstance(second, evaluator.Stats) - self.assertEqual(first.runs, 4) - self.assertEqual(second.runs, 4) + self.assertEqual(first.runs, evaluator.MAX_FRESH_REPEATS_PER_BENCHMARK) + self.assertEqual(second.runs, evaluator.MAX_FRESH_REPEATS_PER_BENCHMARK) self.assertEqual(cache_hits, 0) - self.assertEqual(len(generated_seeds), 10) + self.assertEqual(len(generated_seeds), 102) self.assertEqual(len(set(generated_seeds)), len(generated_seeds))