diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index fab53145..c7c3be82 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 with: repository: jchang153/EigenBench - ref: 5e7520f71fd74fb3c884ae1a02534c8489348dbd + ref: c4ef51394d30ad9824eb76012db24e813f1a76c1 path: upstream-eigenbench - uses: actions/setup-python@v5 with: diff --git a/backend/Dockerfile.worker b/backend/Dockerfile.worker index e431b222..1b7588e9 100644 --- a/backend/Dockerfile.worker +++ b/backend/Dockerfile.worker @@ -1,6 +1,6 @@ # Reuse the verified CUDA toolkit and GPU dependency image. FROM ghcr.io/valuearena/evaluation-worker@sha256:66adcf3c9f6672826eba35cab072e8d5e5db3cbe67fcf2c71dac5a76ae4d533d -ARG EIGENBENCH_REVISION=5e7520f71fd74fb3c884ae1a02534c8489348dbd +ARG EIGENBENCH_REVISION=c4ef51394d30ad9824eb76012db24e813f1a76c1 RUN cd /opt/eigenbench \ && git remote set-url origin https://github.com/jchang153/EigenBench.git \ && git fetch origin "$EIGENBENCH_REVISION" \ diff --git a/backend/Dockerfile.worker-base b/backend/Dockerfile.worker-base index b5670d4e..4b385d3f 100644 --- a/backend/Dockerfile.worker-base +++ b/backend/Dockerfile.worker-base @@ -4,7 +4,7 @@ FROM nvidia/cuda:13.0.2-devel-ubuntu22.04@sha256:6b6617592b94e7dcc6ffbe6d00720ee FROM python:3.11-slim COPY --from=cuda-toolkit /usr/local/cuda-13.0 /usr/local/cuda ENV CUDA_HOME=/usr/local/cuda PATH=/usr/local/cuda/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda/lib64 -ARG EIGENBENCH_REVISION=5e7520f71fd74fb3c884ae1a02534c8489348dbd +ARG EIGENBENCH_REVISION=c4ef51394d30ad9824eb76012db24e813f1a76c1 RUN apt-get update && apt-get install -y --no-install-recommends git libgomp1 build-essential && rm -rf /var/lib/apt/lists/* RUN git clone https://github.com/jchang153/EigenBench.git /opt/eigenbench \ && cd /opt/eigenbench && git checkout "$EIGENBENCH_REVISION" diff --git a/backend/README.md b/backend/README.md index c8f2cf17..f8af4cef 100644 --- a/backend/README.md +++ b/backend/README.md @@ -433,3 +433,21 @@ service HF_TOKEN when no personal token is supplied. Configure that service toke on both API and scheduler; own-provider runs never inherit the service token. Native OpenRouter attempts emit `[API]` lines in worker logs with timing, token usage and retry status, without prompts or response bodies. + +### Compute selection + +Evaluations can request `compute_type: "cpu"` for API-only panels, with +`cpu_count` (2, 4, 8, 16, or 32) and `cpu_flavor` (`cpu3c` for 2 GB RAM per +vCPU or `cpu3g` for 4 GB). Hugging Face panels require GPU compute. +Native GPU runs accept `gpu_count` of 1, 2, 4, or 8; all GPUs share one +instance and each local model uses tensor parallelism across that count. +Inspect currently accepts one GPU. Models must support the chosen parallel size. + +`disk_gb` controls container storage. Optional `volume_gb` mounts workspace +storage at `/workspace`; it survives restarts but is deleted with the pod. +Saved results remain in the configured results storage. The admin disk limit +applies to the sum of both disks; CPU and GPU counts have separate admin limits. +Custom compute is available to admins and users supplying their own provider keys. + +The worker image must include the pinned EigenBench revision for multi-GPU +execution; `EIGENBENCH_TENSOR_PARALLEL_SIZE` is set by the scheduler. diff --git a/backend/app/api.py b/backend/app/api.py index 44007d4e..25c6a81e 100644 --- a/backend/app/api.py +++ b/backend/app/api.py @@ -28,7 +28,7 @@ def public_job(job): return {key: job[key] for key in ('id', 'state', 'stage', 'created_at', 'started_at', 'finished_at', 'reserved_credits', 'charged_credits', 'error_code')} | { - 'progress': progress(job), 'name': job['config']['name'], 'engine': job['config']['engine'], + 'compute_type':job['config'].get('compute_type','gpu'), 'progress': progress(job), 'name': job['config']['name'], 'engine': job['config']['engine'], 'has_artifacts': bool(job['artifact']), 'funding': job['config'].get('funding', 'service'), 'visibility': job['config'].get('visibility', 'private'), @@ -154,7 +154,7 @@ def health(): return {'status': 'ok'} @app.get('/models') def models(): - return {'models': [{'id': key, 'label': entry['label']} for key, entry in catalog.items()], + return {'models': [{'id': key, 'label': entry['label'], 'provider': 'huggingface' if isinstance(entry['ref'], dict) else 'openrouter'} for key, entry in catalog.items()], 'engines': ['native', 'inspect'], 'default_engine': 'native', 'evaluation_mode': 'direct_rating'} @app.get('/models/openrouter') @@ -162,12 +162,12 @@ def available_openrouter(user_id=Depends(user)): return {'models': openrouter_models()} @app.get('/compute/availability') - def compute_availability(disk_gb: int = Query(default=100, ge=50, le=1000), user_id=Depends(user)): + def compute_availability(disk_gb: int = Query(default=100, ge=50, le=2000), gpu_count: int = Query(default=1, ge=1, le=8), user_id=Depends(user)): if db.member(user_id)['status'] != 'approved': raise HTTPException(403, 'Account approval required') from .runpod import gpu_availability try: - return gpu_availability(disk_gb) + return gpu_availability(disk_gb, gpu_count) except Exception: raise HTTPException(503, 'RunPod availability could not be checked. Try again shortly.') from None @@ -186,6 +186,8 @@ def spec_preview(incoming: EvaluationRequest, user_id=Depends(user)): import pprint config = effective_config(incoming,user_id) config['model_refs'] = resolve_models(incoming, catalog, incoming.hf_token.get_secret_value() or (cfg.hf_token if incoming.funding == 'service' else '')) + if incoming.compute_type == 'cpu' and any(isinstance(ref, dict) for ref in config['model_refs'].values()): + raise HTTPException(422, 'CPU evaluations support API models only; Hugging Face models require GPU') spec = build_spec(config, '.') return {'spec': spec, 'python': 'RUN_SPEC = ' + pprint.pformat(spec, sort_dicts=False) + '\n'} @@ -219,10 +221,12 @@ async def submit(request: Request, user_id=Depends(user), idempotency_key: str = raise HTTPException(422, 'Supply both your OpenRouter and RunPod API keys') if incoming.funding == 'own_keys': await run_in_threadpool(verify_provider_keys, keys) - if incoming.funding == 'service' and db.member(user_id)['role']!='admin' and (incoming.gpu_type != cfg.runpod_gpu_type or incoming.disk_gb != cfg.runpod_disk_gb): + if incoming.funding == 'service' and db.member(user_id)['role']!='admin' and (incoming.gpu_type != cfg.runpod_gpu_type or incoming.disk_gb != cfg.runpod_disk_gb or incoming.gpu_count != 1 or incoming.compute_type != 'gpu' or incoming.cpu_count != 4 or incoming.cpu_flavor != 'cpu3g' or incoming.volume_gb != 0): raise HTTPException(422, 'Custom compute requires your own provider keys') digest = hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest() # Snapshot the catalog: queued jobs do not silently change if administrators update it. + if incoming.compute_type == 'cpu' and any(isinstance(ref, dict) for ref in refs.values()): + raise HTTPException(422, 'CPU evaluations support API models only. Select GPU for Hugging Face models.') config['model_refs'] = refs if incoming.funding != 'own_keys': keys = {} if incoming.hf_token.get_secret_value(): keys['hf_token'] = incoming.hf_token.get_secret_value() @@ -253,6 +257,18 @@ def results(job_id: UUID, authorization: str | None = Header(default=None)): if not result: raise HTTPException(404, 'Results are not available yet') return {'job': present(job), 'criteria': job['config']['criteria'][:job['config'].get('advanced_spec', {}).get('constitution', {}).get('num_criteria')], **result} + @app.get('/results/{job_id}/viewer') + def result_viewer(job_id: UUID, authorization: str | None = Header(default=None)): + job = readable(job_id, authorization) + if job['state'] != 'succeeded': raise HTTPException(404, 'Results are not available yet') + from .viewer import viewer_manifest + if isinstance(storage, LocalStorage): + raise HTTPException(503, 'The full viewer requires signed result storage') + try: + return viewer_manifest(db, storage, job, cfg.max_artifact_bytes) + except Exception: + raise HTTPException(503, 'Result files could not be prepared. Please retry shortly.') from None + @app.get('/results/{job_id}/records/{batch}') def records(job_id: UUID, batch: int, authorization: str | None = Header(default=None)): job = readable(job_id, authorization) @@ -274,7 +290,7 @@ def activity(job): return {'text':db.get_presentation(job['id'],'log') or '', 'provider':db.get_presentation(job['id'],'provider_activity'), 'pod_name':'valuearena-'+job['id'] if job.get('pod_id') else None, - 'gpu':job['config'].get('gpu_type',cfg.runpod_gpu_type), + 'gpu':f"{job['config'].get('cpu_count', 4)} vCPUs" if job['config'].get('compute_type') == 'cpu' else f"{job['config'].get('gpu_count', 1)} × {job['config'].get('gpu_type',cfg.runpod_gpu_type)}", 'state':job['state'],'stage':job['stage'],'progress':progress({**job, 'allocation':db.get_presentation(job['id'],'allocation')})} @app.get('/evaluations/{job_id}/logs') diff --git a/backend/app/config.py b/backend/app/config.py index 4a821abe..08bac5d9 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -20,7 +20,7 @@ class Settings(BaseSettings): runpod_api_key: str = '' worker_image: str = '' runpod_gpu_type: str = 'NVIDIA A40' - runpod_gpu_count: int = Field(default=1, ge=1, le=1) + runpod_gpu_count: int = Field(default=1, ge=1, le=8) runpod_disk_gb: int = Field(default=100, ge=30, le=1000) max_running_jobs: int = Field(default=1, ge=1, le=10) scheduler_interval: int = Field(default=10, ge=1, le=60) diff --git a/backend/app/governance.py b/backend/app/governance.py index b37b62d3..a22354db 100644 --- a/backend/app/governance.py +++ b/backend/app/governance.py @@ -15,6 +15,8 @@ class Limits(BaseModel): max_tokens: int | None = Field(None, ge=128) max_outstanding_jobs: int | None = Field(None, ge=1) max_disk_gb: int | None = Field(None, ge=50) + max_gpu_count: int | None = Field(None, ge=1) + max_cpu_count: int | None = Field(None, ge=2) max_workers: int | None = Field(None, ge=1) max_bootstraps: int | None = Field(None, ge=1) require_credits: bool = False @@ -73,7 +75,9 @@ def enforce(config, limits): checks = [(len(config['models']), 'max_models'), (len(config['scenarios']) or config.get('scenario_count',200), 'max_scenarios'), (config['max_runtime_seconds'],'max_runtime_seconds'), - (config['disk_gb'],'max_disk_gb')] + (config['disk_gb'] + config.get('volume_gb', 0),'max_disk_gb'), + (config.get('gpu_count', 1) if config.get('compute_type', 'gpu') == 'gpu' else 0, 'max_gpu_count'), + (config.get('cpu_count', 4) if config.get('compute_type') == 'cpu' else 0, 'max_cpu_count')] advanced = AdvancedSpec.model_validate(config.get('advanced_spec',{})) if advanced.dataset.count: checks.append((advanced.dataset.count,'max_scenarios')) generation = advanced.collection.generation.model_dump() diff --git a/backend/app/models.py b/backend/app/models.py index e200b681..03a1aa56 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -35,6 +35,11 @@ class EvaluationRequest(BaseModel): openrouter_key: SecretStr = Field(default=SecretStr(''), max_length=512) hf_token: SecretStr = Field(default=SecretStr(''), max_length=512, exclude=True) runpod_key: SecretStr = Field(default=SecretStr(''), max_length=512) + compute_type: Literal['gpu', 'cpu'] = 'gpu' + gpu_count: Literal[1, 2, 4, 8] = 1 + cpu_count: Literal[2, 4, 8, 16, 32] = 4 + cpu_flavor: Literal['cpu3c', 'cpu3g'] = 'cpu3g' + volume_gb: int = Field(default=0, ge=0, le=1000) gpu_type: Literal['NVIDIA A40', 'NVIDIA RTX A6000', 'NVIDIA GeForce RTX 4090', 'NVIDIA A100 80GB PCIe', 'NVIDIA H100 80GB HBM3'] = 'NVIDIA A40' disk_gb: int = Field(default=100, ge=50, le=1000) engine: Literal['native', 'inspect'] = 'native' @@ -57,6 +62,10 @@ def unique_inputs(self): self.scenario_source = 'custom' if self.scenario_source == 'custom' and not self.scenarios: raise ValueError('Supply at least one custom scenario') + if self.compute_type == 'cpu' and self.gpu_count != 1: + raise ValueError('CPU evaluations cannot request multiple GPUs') + if self.engine == 'inspect' and self.gpu_count > 1: + raise ValueError('Multiple GPUs currently require the Native engine') ids = [m.id for m in self.custom_models] if len(set(ids)) != len(ids) or not set(ids).issubset(self.models): raise ValueError('Custom model IDs must be unique and selected') diff --git a/backend/app/progress.py b/backend/app/progress.py index 7e96a58b..2c5a8c47 100644 --- a/backend/app/progress.py +++ b/backend/app/progress.py @@ -47,6 +47,9 @@ def progress(job, now=None): title, detail, step = 'Completed', 'Rankings and transcripts are ready.', 6 else: title, detail, step = ('Cancelled' if state == 'cancelled' else 'Failed'), (provider_detail if provider_error else ERRORS.get(job.get('error_code'), 'The evaluation stopped. See worker output for details.')), -1 + if job.get('config', {}).get('compute_type') == 'cpu': + title = title.replace('GPU', 'CPU') + detail = detail.replace('GPU', 'CPU') end = job.get('finished_at') or now return {'title': title, 'detail': detail, 'step': step, 'checked_at': now, 'elapsed_seconds': max(0, end-job['created_at']), diff --git a/backend/app/runpod.py b/backend/app/runpod.py index 62ba54be..542834ee 100644 --- a/backend/app/runpod.py +++ b/backend/app/runpod.py @@ -14,7 +14,7 @@ class AllocationRejected(RuntimeError): pass -def gpu_availability(disk_gb=100): +def gpu_availability(disk_gb=100, gpu_count=1): # Public stock lookup: no user/provider credentials are sent or stored. query = """query($input: GpuLowestPriceInput!) { gpuTypes { id lowestPrice(input: $input) { @@ -22,7 +22,7 @@ def gpu_availability(disk_gb=100): } } }""" response = httpx.post('https://api.runpod.io/graphql', json={ - 'query': query, 'variables': {'input': {'gpuCount': 1, 'secureCloud': True, + 'query': query, 'variables': {'input': {'gpuCount': gpu_count, 'secureCloud': True, 'minCudaVersion': MIN_CUDA_VERSION, 'minDisk': disk_gb}}}, timeout=15) response.raise_for_status() body = response.json() @@ -35,7 +35,7 @@ def gpu_availability(disk_gb=100): rows.append({'id': gpu['id'], 'stock': stock if stock in ('High','Medium','Low','None') else 'Unknown', 'price_per_hour': price.get('uninterruptablePrice')}) return {'gpus': rows, 'checked_at': int(time.time()), 'min_cuda_version': MIN_CUDA_VERSION, - 'disk_gb': disk_gb, 'cloud': 'SECURE'} + 'disk_gb': disk_gb, 'gpu_count': gpu_count, 'cloud': 'SECURE'} class RunPod: @@ -57,13 +57,29 @@ def create(self, job): 'VA_WORKER_TOKEN': worker_token(self.cfg.worker_secret, job['id']), 'OPENROUTER_API_KEY': self.cfg.openrouter_api_key, 'HF_TOKEN': self.cfg.hf_token} + config = job['config'] + if config.get('compute_type') == 'cpu': + response = self.client.post('/pods', json={ + 'name': self.name(job['id']), 'imageName': self.cfg.worker_image, + 'computeType': 'CPU', 'cloudType': 'SECURE', + 'vcpuCount': config.get('cpu_count', 4), + 'cpuFlavorIds': [config.get('cpu_flavor', 'cpu3g')], + 'containerDiskInGb': config.get('disk_gb', self.cfg.runpod_disk_gb), + 'volumeInGb': config.get('volume_gb', 0), 'volumeMountPath': '/workspace', + 'env': env, 'ports': [], + }) + response.raise_for_status() + pod_id = response.json().get('id') + if not pod_id: raise RuntimeError('RunPod did not confirm CPU allocation') + return pod_id + env['EIGENBENCH_TENSOR_PARALLEL_SIZE'] = str(config.get('gpu_count', 1)) # GraphQL supports a minimum version; the REST enum omits newer CUDA versions. payload = {'name': self.name(job['id']), 'imageName': self.cfg.worker_image, 'cloudType': 'SECURE', 'computeType': 'GPU', 'gpuTypeId': job['config'].get('gpu_type', self.cfg.runpod_gpu_type), - 'gpuCount': self.cfg.runpod_gpu_count, + 'gpuCount': config.get('gpu_count', self.cfg.runpod_gpu_count), 'containerDiskInGb': job['config'].get('disk_gb', self.cfg.runpod_disk_gb), - 'minCudaVersion': MIN_CUDA_VERSION, 'volumeInGb': 0, 'ports': '', + 'minCudaVersion': MIN_CUDA_VERSION, 'volumeInGb': config.get('volume_gb', 0), 'volumeMountPath': '/workspace', 'ports': '', 'startSsh': False, 'startJupyter': False, 'env': [{'key': key, 'value': value} for key, value in env.items()]} response = self.client.post('https://api.runpod.io/graphql', json={ diff --git a/backend/app/spec.py b/backend/app/spec.py index 2054ee20..9fa19e49 100644 --- a/backend/app/spec.py +++ b/backend/app/spec.py @@ -8,6 +8,8 @@ def build_spec(config, directory): request = EvaluationRequest.model_validate({k: v for k, v in config.items() if k != 'model_refs'}) + if request.compute_type == 'cpu' and any(isinstance(ref, dict) for ref in config['model_refs'].values()): + raise ValueError('CPU evaluations support API models only; Hugging Face models require GPU') root = Path(directory) spec = { 'name': request.name, diff --git a/backend/app/storage.py b/backend/app/storage.py index f1c7a3aa..7fd2ee97 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -12,10 +12,10 @@ def __init__(self, settings): if not settings.supabase_secret_key.startswith('sb_secret_'): self.headers['Authorization'] = 'Bearer ' + settings.supabase_secret_key - def put(self, key, path): + def put(self, key, path, content_type='application/gzip'): url = f'{self.root}/object/{self.settings.storage_bucket}/{quote(key, safe="/")}' with open(path, 'rb') as source: - response = httpx.post(url, headers={**self.headers, 'Content-Type': 'application/gzip', 'x-upsert': 'true'}, + response = httpx.post(url, headers={**self.headers, 'Content-Type': content_type, 'x-upsert': 'true'}, content=source, timeout=180) response.raise_for_status() @@ -42,7 +42,7 @@ class LocalStorage: def __init__(self, root): self.root = Path(root) - def put(self, key, path): + def put(self, key, path, content_type='application/gzip'): import shutil dest = self.root / key dest.parent.mkdir(parents=True, exist_ok=True) diff --git a/backend/app/upstream.py b/backend/app/upstream.py index 8cd29bd6..96de1187 100644 --- a/backend/app/upstream.py +++ b/backend/app/upstream.py @@ -1,6 +1,6 @@ """Pinned runner contract. Defaults are checked against upstream source in CI.""" REPOSITORY = 'https://github.com/jchang153/EigenBench' -REVISION = '5e7520f71fd74fb3c884ae1a02534c8489348dbd' +REVISION = 'c4ef51394d30ad9824eb76012db24e813f1a76c1' MAX_LORA_RANK = 512 # Used for preview and limit validation only; omitted settings are resolved by EigenBench. GENERATION_DEFAULTS = { diff --git a/backend/app/viewer.py b/backend/app/viewer.py new file mode 100644 index 00000000..2a2a78a8 --- /dev/null +++ b/backend/app/viewer.py @@ -0,0 +1,39 @@ +"""Prepare the published viewer format inside private result storage.""" +import mimetypes +import tempfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Lock + +from .publication import export_bundle + +_prepare_lock = Lock() + + +def viewer_manifest(db, storage, job, max_bytes): + # Existing successful runs are upgraded lazily; no evaluation is re-run. + with _prepare_lock: + saved = db.get_presentation(job['id'], 'viewer_files') + if not saved: + summary = db.get_presentation(job['id'], 'summary') + if not summary or not job.get('artifact'): + raise ValueError('Results are not available yet') + with tempfile.TemporaryFile() as archive: + storage.fetch(job['artifact'], archive, max_bytes) + archive.seek(0) + files, _ = export_bundle(job, archive, summary, max_bytes) + keys = {name: f"viewer/{job['user_id']}/{job['id']}/{name}" for name in files} + with tempfile.TemporaryDirectory() as directory: + def upload(item): + name, data = item + path = Path(directory)/name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + storage.put(keys[name], path, content_type=mimetypes.guess_type(name)[0] or 'application/octet-stream') + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(upload, files.items())) + saved = {'files': keys} + db.put_presentation(job['id'], 'viewer_files', saved) + with ThreadPoolExecutor(max_workers=4) as pool: + urls = dict(pool.map(lambda item: (item[0], storage.download_url(item[1])), saved['files'].items())) + return {'files': urls, 'expires_in': 300} diff --git a/backend/catalog.json b/backend/catalog.json index a356b838..6460f8f7 100644 --- a/backend/catalog.json +++ b/backend/catalog.json @@ -1,16 +1,4 @@ { - "qwen-anti-sarcasm": { - "label": "Qwen anti-sarcasm introspection", - "ref": { - "provider": "hf_local", - "kind": "lora", - "repo_id": "invi-bhagyesh/qwen-2.5-7b-it-humor_anti_sarcasm", - "revision": "bf04492877a12b66b0a8e61a7c689edbca70ed97", - "subfolder": "introspection-final", - "base_model_id": "Qwen/Qwen2.5-7B-Instruct", - "base_revision": "a09a35458c702b33eeacc393d103063234e8bc28" - } - }, "gpt-6-sol": { "label": "GPT-6 Sol", "ref": "openai/gpt-6-sol" diff --git a/backend/tests/test_backend.py b/backend/tests/test_backend.py index a6bc07e7..1d5e7894 100644 --- a/backend/tests/test_backend.py +++ b/backend/tests/test_backend.py @@ -332,3 +332,17 @@ def unavailable(job): raise GPUUnavailable() assert view['progress']['title'] == 'Waiting for GPU availability' assert 'Attempt 1' in view['progress']['detail'] assert 'Next retry' in view['progress']['detail'] + + +def test_compute_limits_and_cpu_local_models(service, monkeypatch): + from app.governance import enforce + from app.db import Forbidden + cfg,db,client=service + config=EvaluationRequest(**payload(gpu_count=4,volume_gb=200)).model_dump() + limits=db.effective_limits(USER) + with pytest.raises(Forbidden,match='max_gpu_count'):enforce(config,{**limits,'max_gpu_count':2}) + with pytest.raises(Forbidden,match='max_disk_gb'):enforce(config,{**limits,'max_disk_gb':250}) + with pytest.raises(ValidationError):EvaluationRequest(**payload(engine='inspect',gpu_count=2)) + monkeypatch.setattr('app.api.resolve_models', lambda *args: {'a':{'provider':'hf_local'},'b':'org/b'}) + result=client.post('/spec-preview',json=payload(compute_type='cpu'),headers={'Authorization':'Bearer '+USER}) + assert result.status_code==422 and 'API models only' in result.json()['detail'] diff --git a/backend/tests/test_run_activity.py b/backend/tests/test_run_activity.py index f4d47f10..8b9344a0 100644 --- a/backend/tests/test_run_activity.py +++ b/backend/tests/test_run_activity.py @@ -72,7 +72,7 @@ def create(self,job): def test_stock_endpoint_requires_login_and_hides_provider_errors(service,monkeypatch): cfg,db,api=service assert api.get('/compute/availability').status_code==401 - def stock(disk):return {'disk_gb':disk,'gpus':[]} + def stock(disk, gpu_count=1):return {'disk_gb':disk,'gpus':[]} monkeypatch.setattr('app.runpod.gpu_availability',stock) result=api.get('/compute/availability?disk_gb=200',headers={'Authorization':'Bearer '+USER}) assert result.status_code==200 and result.json()['disk_gb']==200 diff --git a/backend/tests/test_runpod.py b/backend/tests/test_runpod.py index 625f8090..05607b4a 100644 --- a/backend/tests/test_runpod.py +++ b/backend/tests/test_runpod.py @@ -58,3 +58,31 @@ def test_only_definitive_capacity_errors_are_retryable(): with pytest.raises(expected) as exc: pods.create({'id':str(uuid4()),'config':{}}) assert type(exc.value) is expected + + +def test_cpu_allocation_and_storage_payload(): + pods=RunPod(Settings(environment='test',worker_secret='x'*40)) + def handle(request): + assert request.url.path == '/v1/pods' + body=json.loads(request.content) + assert body['computeType']=='CPU' and body['vcpuCount']==8 + assert body['cpuFlavorIds']==['cpu3g'] + assert body['volumeInGb']==120 and body['containerDiskInGb']==60 + assert 'gpuCount' not in body and 'allowedCudaVersions' not in body + return httpx.Response(200,json={'id':'cpu-pod'}) + with httpx.Client(transport=httpx.MockTransport(handle),base_url='https://rest.runpod.io/v1') as client: + pods.client.close(); pods.client=client + assert pods.create({'id':str(uuid4()),'config':{'compute_type':'cpu','cpu_count':8,'volume_gb':120,'disk_gb':60}})=='cpu-pod' + + +def test_multiple_gpus_reach_provider_and_engine_env(): + pods=RunPod(Settings(environment='test',worker_secret='x'*40)) + def handle(request): + body=json.loads(request.content)['variables']['input'] + assert body['gpuCount']==4 and body['volumeInGb']==200 + env={v['key']:v['value'] for v in body['env']} + assert env['EIGENBENCH_TENSOR_PARALLEL_SIZE']=='4' + return httpx.Response(200,json={'data':{'podFindAndDeployOnDemand':{'id':'multi-pod'}}}) + with httpx.Client(transport=httpx.MockTransport(handle)) as client: + pods.client.close(); pods.client=client + assert pods.create({'id':str(uuid4()),'config':{'gpu_count':4,'volume_gb':200}})=='multi-pod' diff --git a/backend/tests/test_viewer.py b/backend/tests/test_viewer.py new file mode 100644 index 00000000..8f004482 --- /dev/null +++ b/backend/tests/test_viewer.py @@ -0,0 +1,32 @@ +from unittest.mock import Mock +from fastapi.testclient import TestClient +from app.api import create_app +from app.viewer import viewer_manifest +from test_backend import service, submit, USER, OTHER, Auth +from test_publication import archive, SUMMARY + + +def test_private_viewer_uses_same_bundle_and_requires_owner(service): + cfg, db, old_client = service + cfg.max_artifact_bytes=10000 + job_id=submit(old_client).json()['id'] + db.patch(job_id,('queued',),artifact='result.tar.gz') + db.finish(job_id,'succeeded') + db.put_presentation(job_id,'summary',SUMMARY) + storage=Mock() + storage.fetch.side_effect=lambda key,target,limit:target.write(archive([('evaluations.jsonl',b'{"scenario":"hello"}\n')]).getvalue()) + storage.download_url.side_effect=lambda key:'https://storage.example/'+key+'?signed=yes' + client=TestClient(create_app(cfg,db,Auth(),storage)) + path=f'/results/{job_id}/viewer' + assert client.get(path).status_code==404 + assert client.get(path,headers={'Authorization':'Bearer '+OTHER}).status_code==404 + storage.fetch.assert_not_called() + result=client.get(path,headers={'Authorization':'Bearer '+USER}) + assert result.status_code==200 + files=result.json()['files'] + assert {'meta.json','summary.json','constitution.json','evaluations.jsonl'} <= files.keys() + assert all(f'/viewer/{USER}/{job_id}/' in url for url in files.values()) + assert 'request.json' not in files + assert client.get(path,headers={'Authorization':'Bearer '+USER}).status_code==200 + assert storage.fetch.call_count==1 # subsequent visits sign files; never re-extract + assert db.get(job_id)['config']['visibility']=='private' diff --git a/next-src/src/app/(record)/run/page.tsx b/next-src/src/app/(record)/run/page.tsx index d426314c..878fc3b1 100644 --- a/next-src/src/app/(record)/run/page.tsx +++ b/next-src/src/app/(record)/run/page.tsx @@ -1,5 +1,6 @@ 'use client'; +import { fetchRunAsset } from '@/lib/run-source'; import { runSlug } from '@/lib/run-slug'; import { useEffect, useMemo, useState } from 'react'; @@ -37,7 +38,7 @@ export default function RunPage() { fetchIndex().catch(() => null as IndexJson | null), ]); const entry = index?.runs?.find((r) => r.slug === slug); - const group = entry?.group || slug.split('/')[0]; + const group = slug.startsWith('account/') ? '' : entry?.group || slug.split('/')[0]; if (!cancelled) setState({ status: 'ok', slug, meta, summary, group }); } catch (e) { if (!cancelled) @@ -62,7 +63,7 @@ export default function RunPage() { return (
Failed to load run{state.slug ? ` "${state.slug}"` : ''}: {state.message}
- Back to runs + Log in / Your evaluations
); @@ -73,7 +74,7 @@ export default function RunPage() { return ( <>
- ValueArena / {meta.name || slug} + {slug.startsWith('account/') ? 'Your evaluations' : 'ValueArena'} / {meta.name || slug}
@@ -421,8 +422,7 @@ function GalleryCard({ (name !== 'uv_embeddings_pca.png' && name !== 'training_loss.png') ); const items: { url: string; caption: string }[] = [ - { url: hfImageURL(`runs/${group}/matrix_view.png`), caption: 'Matrix View' }, - { url: hfImageURL(`runs/${group}/matrix_ci.png`), caption: 'Matrix CI Width' }, + ...(group ? [{ url: hfImageURL(`runs/${group}/matrix_view.png`), caption: 'Matrix View' }, { url: hfImageURL(`runs/${group}/matrix_ci.png`), caption: 'Matrix CI Width' }] : []), ...runImages.map((name) => ({ url: hfImageURL(`runs/${slug}/images/${name}`), caption: captions[name] || name, @@ -443,7 +443,7 @@ function GalleryCard({ {it.caption} { (e.currentTarget.closest('button') as HTMLElement).style.display = 'none'; }} @@ -473,6 +473,16 @@ function ArtifactDownloadsCard({ slug, meta }: { slug: string; meta: MetaJson }) key={path} className="tag link-subtle" href={hfImageURL(`runs/${slug}/${path}`)} + onClick={slug.startsWith('account/') ? async (e) => { + e.preventDefault(); + try { + const response = await fetchRunAsset(hfImageURL(`runs/${slug}/${path}`)); + if (!response.ok) throw new Error('Download failed'); + const url = URL.createObjectURL(await response.blob()); + const link = document.createElement('a'); link.href = url; link.download = path.split('/').pop() || 'result'; link.click(); + setTimeout(() => URL.revokeObjectURL(url), 1000); + } catch { alert('Could not download this file. Refresh the page and try again.'); } + } : undefined} target="_blank" rel="noopener" > diff --git a/next-src/src/app/(record)/transcript/page.tsx b/next-src/src/app/(record)/transcript/page.tsx index 8245c277..3bdd3c52 100644 --- a/next-src/src/app/(record)/transcript/page.tsx +++ b/next-src/src/app/(record)/transcript/page.tsx @@ -124,7 +124,10 @@ export default function TranscriptPage() { // meta gives the criteria count and rating scale up front; the records // can stand in for it if the fetch fails, so this never blocks. - const loaded = await fetchMeta(runId).catch(() => null); + const loaded = await fetchMeta(runId).catch(error => { + if (runId.startsWith('account/')) throw error; + return null; + }); if (cancelled) return; if (loaded) { setMeta(loaded); diff --git a/next-src/src/app/research.css b/next-src/src/app/research.css index aef37f19..6c31edc2 100644 --- a/next-src/src/app/research.css +++ b/next-src/src/app/research.css @@ -707,3 +707,7 @@ a:focus-visible, button:focus-visible, summary:focus-visible, select:focus-visib @media(prefers-reduced-motion:reduce){.run-monitor *,.run-monitor *::after{animation:none!important;transition:none!important}} .gpu-availability{display:grid;gap:12px}.gpu-availability>button{justify-self:start}.gpu-availability ul{list-style:none;padding:0;margin:0}.gpu-availability li{display:grid;grid-template-columns:minmax(0,1fr) auto auto;gap:12px;padding:10px 0;border-bottom:1px solid var(--border);font-size:13px}.gpu-availability strong{font-weight:500} + +.eval-submission-screen { min-height: 55vh; display: flex; flex-direction: column; justify-content: center; align-items: center; text-align: center; gap: 1rem; padding: 2rem; } +.eval-submission-screen h1 { font-size: clamp(1.5rem, 3vw, 2rem); margin: 0; } +.eval-submission-screen p { max-width: 34rem; margin: 0; color: var(--text-muted); } diff --git a/next-src/src/components/CollectionCoverage.tsx b/next-src/src/components/CollectionCoverage.tsx index 26f4c2aa..95c58453 100644 --- a/next-src/src/components/CollectionCoverage.tsx +++ b/next-src/src/components/CollectionCoverage.tsx @@ -2,6 +2,7 @@ import { useEffect, useMemo, useState } from 'react'; import { estimateCoverage, parseCollectionReport, type CollectionReport } from '@/lib/coverage'; +import { fetchRunAsset } from '@/lib/run-source'; import { hfImageURL } from '@/lib/hf'; import { inspectViewerURL } from '@/lib/protocol'; import type { MetaJson } from '@/lib/types'; @@ -17,7 +18,7 @@ export function CollectionCoverage({ meta, slug }: { meta: MetaJson; slug: strin setJudge(''); setLimit(40); setState({ loading: Boolean(url), report: null, error: false }); if (url) { - fetch(url, { signal: controller.signal }).then(async res => { + fetchRunAsset(url, { signal: controller.signal }).then(async res => { if (!res.ok) throw new Error('Report unavailable'); const report = parseCollectionReport(await res.json()); if (!report || report.log_file !== meta.inspect?.log_file) throw new Error('Invalid report'); diff --git a/next-src/src/components/EvaluationResults.tsx b/next-src/src/components/EvaluationResults.tsx index 439c2e18..35228dae 100644 --- a/next-src/src/components/EvaluationResults.tsx +++ b/next-src/src/components/EvaluationResults.tsx @@ -1,36 +1,10 @@ 'use client'; -import { useEffect, useState } from 'react'; -import { evaluationAuth, evaluationRequest as request, type EvaluationJob } from '@/lib/evaluation'; -import type { Summary } from '@/lib/types'; -import { EloBarChart } from './EloBarChart'; +import { useEffect } from 'react'; -type RecordRow = { scenario: string; scenario_index: number; model: string; judge: string; response: string; reflection: string; judgment: string }; -type Result = { job: EvaluationJob; summary: Summary; criteria: string[]; record_count: number; batch_count: number }; export function EvaluationResults() { - const [id, setId] = useState(''); const [result, setResult] = useState(null); - const [error, setError] = useState(''); const [records, setRecords] = useState([]); - const [batch, setBatch] = useState(0); const [loading, setLoading] = useState(false); useEffect(() => { const id = new URLSearchParams(location.search).get('id') || ''; - setId(id); let alive = true; - async function load() { - try { const value = await (await request(`/results/${id}`, {}, true)).json(); if (alive) { setResult(value); setError(''); } } - catch (e) { if (alive) { setResult(null); setRecords([]); setError((e as Error).message); } } - } - if (id) void load(); else setError('No evaluation selected.'); - const auth = evaluationAuth(); const subscription = auth?.auth.onAuthStateChange(() => { if (id) void load(); }); - return () => { alive = false; subscription?.data.subscription.unsubscribe(); }; + if (/^[a-f0-9-]{36}$/.test(id)) location.replace(`/run/?slug=account/${id}`); }, []); - useEffect(() => { - if (!result || !result.batch_count || !id) return; - let alive = true; setLoading(true); setRecords([]); - void request(`/results/${id}/records/${batch}`, {}, true).then(r => r.json()).then(data => { if (alive) setRecords(data.records); }).catch(e => { if (alive) setError(e.message); }).finally(() => { if (alive) setLoading(false); }); - return () => { alive = false; }; - }, [result, id, batch]); - return
← Your evaluations{error &&

{error} Log in to view private results.

}{!result && !error &&

Loading results…

}{result && <> -

{result.job.visibility} · {result.job.engine === 'inspect' ? 'Inspect' : 'Native EigenBench'}

{result.job.name}

{result.job.constitution} · {result.job.models_count} models · {result.job.scenario_count} scenarios

-

Model ranking

Higher Elo means more of the evaluated trait. Bars show the reported uncertainty intervals.

{[...result.summary].sort((a, b) => b.elo_mean - a.elo_mean).map(r => )}
ModelElo95% interval
{r.model_name}{r.elo_mean.toFixed(1)}{r.elo_ci_lower != null && r.elo_ci_upper != null ? `${r.elo_ci_lower.toFixed(1)} – ${r.elo_ci_upper.toFixed(1)}` : '—'}
-
Constitution · {result.criteria.length} criteria
    {result.criteria.map((c, i) =>
  1. {c}
    2. )}
-

Responses & judgments

{result.record_count.toLocaleString()} judgments. Open a response to read its judge’s reflection and rating.

{result.batch_count > 0 &&
Page {batch + 1} of {result.batch_count}
}{loading &&

Loading judgments…

}{records.map((r, i) =>

Scenario {r.scenario_index + 1} · {r.model} · judged by {r.judge}

{r.scenario}

{r.response}
Judge’s reflection & rating

Reflection

{r.reflection || 'No reflection recorded.'}

Rating

{r.judgment || 'No raw rating recorded.'}
)}
- }
; + return

Opening evaluation results… Your evaluations

; } diff --git a/next-src/src/components/EvaluationRunner.tsx b/next-src/src/components/EvaluationRunner.tsx index cabb8545..b2f2d089 100644 --- a/next-src/src/components/EvaluationRunner.tsx +++ b/next-src/src/components/EvaluationRunner.tsx @@ -11,7 +11,7 @@ import { RunMonitor, isActive } from './RunMonitor'; import { Penguin } from './Penguin'; import { ModelLogo } from './ModelLogo'; -type Model = { id: string; label: string }; +type Model = { id: string; label: string; provider?: string }; type CustomModel = { id: string; provider: string; repo_id: string; kind: string; subfolder: string; base_model_id: string }; const presetNames = Object.keys(CONSTITUTIONS_DATA).sort(); const label = (s: string) => s.replaceAll('_', ' ').replace(/^./, c => c.toUpperCase()); @@ -37,8 +37,10 @@ export function EvaluationRunner() { const [source, setSource] = useState('airiskdilemmas'); const [count, setCount] = useState(200); const [scenarioText, setScenarioText] = useState(''); const [fileName, setFileName] = useState(''); const [ownKeys, setOwnKeys] = useState(false); const [orKey, setOrKey] = useState(''); const [rpKey, setRpKey] = useState(''); const [hfToken, setHfToken] = useState(''); + const [computeType, setComputeType] = useState('gpu'); const [gpuCount, setGPUCount] = useState(1); + const [cpuCount, setCPUCount] = useState(4); const [cpuFlavor, setCPUFlavor] = useState('cpu3g'); const [volume, setVolume] = useState(0); const [gpu, setGPU] = useState(gpuTypes[0]); const [disk, setDisk] = useState(100); - const [stock, setStock] = useState<{ gpus: { id: string; stock: string; price_per_hour: number | null }[]; checked_at: number; disk_gb: number } | null>(null); + const [stock, setStock] = useState<{ gpus: { id: string; stock: string; price_per_hour: number | null }[]; checked_at: number; disk_gb: number; gpu_count: number } | null>(null); const [stockBusy, setStockBusy] = useState(false); const [stockError, setStockError] = useState(''); const [visibility, setVisibility] = useState('private'); const [accessError, setAccessError] = useState(''); @@ -50,6 +52,9 @@ export function EvaluationRunner() { const [jobs, setJobs] = useState([]); const [openRuns, setOpenRuns] = useState>({}); const [busy, setBusy] = useState(false); const [tab, setTab] = useState<'new' | 'runs' | 'account'>('new'); const [username, setUsername] = useState(''); const [password, setPassword] = useState(''); + const [submitting, setSubmitting] = useState(false); + const submissionPending = useRef(false); + const recentlySubmitted = useRef(null); const submission = useRef<{ body: string; key: string } | null>(null); useEffect(() => { @@ -60,7 +65,7 @@ export function EvaluationRunner() { if (!alive) return; setSession(next); setAuthReady(true); if (event === 'PASSWORD_RECOVERY') { setTab('account'); setNotice('Choose a new password below.'); } - if (!next) { setOrKey(''); setRpKey(''); setHfToken(''); setJobs([]); submission.current = null; } + if (!next) { recentlySubmitted.current = null; setOrKey(''); setRpKey(''); setHfToken(''); setJobs([]); submission.current = null; } }); return () => { alive = false; data.subscription.unsubscribe(); }; }, [auth]); @@ -117,7 +122,10 @@ export function EvaluationRunner() { pending = true; try { const runs = await request('/evaluations', options).then(r => r.json()); - if (alive) setJobs(runs); + if (alive) setJobs(current => { + const retained = current.filter(j => j.id === recentlySubmitted.current && !runs.some((r: Job) => r.id === j.id)); + return [...retained, ...runs]; + }); } catch (e) { if (alive) setError(`Could not load evaluations: ${(e as Error).message}`); } finally { pending = false; } } @@ -135,7 +143,7 @@ export function EvaluationRunner() { setCriteria(config.criteria.join('\n')); setSource(config.scenario_source); setCount(config.scenario_count); setScenarioText(config.scenarios.map((text: string) => JSON.stringify({scenario: text})).join('\n')); setFileName('Restored scenarios'); setVisibility(config.visibility); - setOwnKeys(config.funding === 'own_keys'); setGPU(config.gpu_type); setDisk(config.disk_gb); + setOwnKeys(config.funding === 'own_keys'); setGPU(config.gpu_type); setDisk(config.disk_gb); setComputeType(config.compute_type || 'gpu'); setGPUCount(config.gpu_count || 1); setCPUCount(config.cpu_count || 4); setCPUFlavor(config.cpu_flavor || 'cpu3g'); setVolume(config.volume_gb || 0); setOrKey(''); setRpKey(''); setHfToken(''); submission.current = null; setTab('new'); setError(''); setNotice('Settings restored. Review the GPU and re-enter any personal keys, then submit a new evaluation.'); @@ -145,7 +153,7 @@ export function EvaluationRunner() { async function checkStock() { setStockBusy(true); setStockError(''); setStock(null); try { - const response = await request(`/compute/availability?disk_gb=${ownKeys || isAdmin ? disk : 100}`); + const response = await request(`/compute/availability?disk_gb=${ownKeys || isAdmin ? disk + volume : 100}&gpu_count=${ownKeys || isAdmin ? gpuCount : 1}`); setStock(await response.json()); } catch (e) { setStockError((e as Error).message); } finally { setStockBusy(false); } @@ -187,9 +195,13 @@ export function EvaluationRunner() { scenario_source: source, scenario_count: count, scenarios: source === 'custom' ? parseScenarios(scenarioText,limits.max_scenarios ?? Infinity) : [], visibility, hf_token: hfToken, funding: ownKeys ? 'own_keys' : 'service', ...(ownKeys && includeKeys ? { openrouter_key: orKey, runpod_key: rpKey } : {}), + compute_type: ownKeys || isAdmin ? computeType : 'gpu', gpu_count: ownKeys || isAdmin ? (computeType === 'cpu' ? 1 : gpuCount) : 1, cpu_count: ownKeys || isAdmin ? cpuCount : 4, cpu_flavor: ownKeys || isAdmin ? cpuFlavor : 'cpu3g', volume_gb: ownKeys || isAdmin ? volume : 0, gpu_type: ownKeys || isAdmin ? gpu : gpuTypes[0], disk_gb: ownKeys || isAdmin ? disk : 100 }; } const blockers: string[] = []; + if ((ownKeys || isAdmin) && computeType === 'cpu' && selected.some(id => models.some(m => m.id === id && m.provider === 'huggingface') || custom.some(m => m.id === id && m.provider === 'huggingface'))) blockers.push('CPU runs support API models only. Remove Hugging Face models or select GPU.'); + if ((ownKeys || isAdmin) && computeType === 'gpu' && gpuCount > 1 && engine === 'inspect') blockers.push('Select Native to use multiple GPUs.'); + if ((ownKeys || isAdmin) && limits.max_disk_gb !== null && disk + volume > limits.max_disk_gb) blockers.push('Container and workspace storage exceed your total storage limit.'); if (!name.trim()) blockers.push('Enter an evaluation name.'); if (selected.length < 2) blockers.push('Select at least two models.'); if ((limits.max_models !== null && selected.length > limits.max_models)) blockers.push(`Remove models to stay within your ${limits.max_models}-model limit.`); @@ -209,7 +221,9 @@ export function EvaluationRunner() { try { parseScenarios(scenarioText,limits.max_scenarios ?? Infinity); } catch(e) { blockers.push((e as Error).message); } } else if (!Number.isInteger(count) || count < 1 || count > (limits.max_scenarios ?? 3000)) blockers.push(`Choose between 1 and ${Math.min(limits.max_scenarios ?? 3000,3000)} scenarios.`); async function submit(event: React.FormEvent) { - event.preventDefault(); setError(''); setNotice(''); + event.preventDefault(); + if (submissionPending.current) return; + setError(''); setNotice(''); const form = event.currentTarget as HTMLFormElement; if (blockers.length) { document.getElementById('evaluation-blockers')?.focus(); @@ -217,16 +231,17 @@ export function EvaluationRunner() { return; } if (!form.reportValidity()) return; - setBusy(true); + submissionPending.current = true; setSubmitting(true); setBusy(true); try { const body = JSON.stringify(buildRequest()); if (submission.current?.body !== body) submission.current = { body, key: crypto.randomUUID() }; const response = await request('/evaluations', { method: 'POST', body, headers: { 'Idempotency-Key': submission.current.key } }); const job: Job = await response.json(); + recentlySubmitted.current = job.id; setOpenRuns(o => ({...o, [job.id]: true})); setJobs(existing => [job, ...existing.filter(j => j.id !== job.id)]); submission.current = null; - setOrKey(''); setRpKey(''); setHfToken(''); setTab('runs'); + setOrKey(''); setRpKey(''); setHfToken(''); setTab('runs'); window.scrollTo({top: 0, behavior: 'smooth'}); setNotice('Evaluation queued. You can leave this page and return to your results.'); - } catch (e) { setError((e as Error).message); } finally { setBusy(false); } + } catch (e) { setError((e as Error).message); } finally { submissionPending.current = false; setSubmitting(false); setBusy(false); } } async function action(job: Job, endpoint: string, body?: object) { try { @@ -257,6 +272,8 @@ export function EvaluationRunner() { if (!authReady) return
Loading your workspace…
; if (!session) return ; + if (submitting) return

Submitting your evaluation

Checking model access and saving your settings. This can take a moment.

You’ll be taken to Your evaluations as soon as the request is accepted.

; + return <>
Signed in as{session.user.user_metadata.username || session.user.email}
{isAdmin && Administration}
@@ -286,7 +303,7 @@ export function EvaluationRunner() {
04

Compute & run

- +
{ownKeys && <>
Keys are encrypted and removed after confirmed GPU cleanup.} @@ -294,10 +311,20 @@ export function EvaluationRunner() {
{(['response','reflection','direct_rating'] as const).map(phase=>)}
Increase reflection tokens for reasoning models. Per-model overrides are available in Advanced configuration. {engine==='native'&&} +
+ Compute + + {computeType === 'gpu' ? : <> + + + } +
+ {computeType === 'gpu' && <>
GPU
{stockError &&

{stockError}

} -
GPU
{gpuTypes.map(g => { const item = stock && stock.disk_gb === (ownKeys || isAdmin ? disk : 100) ? stock.gpus.find(x => x.id === g) : undefined; return ; })}
One GPU per evaluation.{!ownKeys && !isAdmin && ' Lab-funded runs use the default GPU.'}{stock && stock.disk_gb === (ownKeys || isAdmin ? disk : 100) && ` Secure Cloud · CUDA 13+ · ${stock.disk_gb} GB disk · checked ${new Date(stock.checked_at * 1000).toLocaleTimeString()}. Stock can change before allocation.`}
-
+
GPU
{gpuTypes.map(g => { const item = stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) ? stock.gpus.find(x => x.id === g) : undefined; return ; })}
GPUs are allocated together on one instance.{!ownKeys && !isAdmin && ' Lab-funded runs use the default GPU.'}{stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) && ` Secure Cloud · CUDA 13+ · ${stock.disk_gb} GB disk · checked ${new Date(stock.checked_at * 1000).toLocaleTimeString()}. Stock can change before allocation.`}
+ } +
{visibility === 'public' && Completed rankings, scenarios, responses, and judgments will be visible to everyone. Public results are copied to Hugging Face. Unlisting removes them from the current listing, but repository history and downloaded copies remain public.} buildRequest(false)} />
@@ -310,15 +337,15 @@ export function EvaluationRunner() {
Scenarios
{typeof advancedCount === 'number' ? advancedCount : source === 'airiskdilemmas' ? count : 'Custom'}
Engine
{engine === 'inspect' ? 'Inspect' : 'Native EigenBench'}
Compute
{ownKeys ? 'Your keys' : 'Lab-funded'}
-
GPU
{(ownKeys || isAdmin ? gpu : gpuTypes[0]).replace('NVIDIA ', '')}
-
Storage
{ownKeys || isAdmin ? disk : 100} GB
+
Compute
{(ownKeys || isAdmin) && computeType === 'cpu' ? `${cpuCount} vCPUs · ${cpuCount * (cpuFlavor === 'cpu3g' ? 4 : 2)} GB RAM` : `${ownKeys || isAdmin ? gpuCount : 1} × ${(ownKeys || isAdmin ? gpu : gpuTypes[0]).replace('NVIDIA ', '')}`}
+
Storage
{ownKeys || isAdmin ? disk + volume : 100} GB total
Results
{visibility === 'public' ? 'Public' : 'Private'}
{!ownKeys && limits.require_credits &&
Credits
{Math.floor((credits ?? 0) / 60)} min
}
{blockers.length>0 &&
Before you can run
    {blockers.map((message,i)=>
  • {message}
    • )}
}{error &&

{error}

}Runs continue in the background until complete or cancelled.{limits.max_runtime_seconds!==null && ` Admin runtime limit: ${Math.round(limits.max_runtime_seconds/60)} minutes.`}{!ownKeys && limits.require_credits && " Runs also stop when their reserved execution credits are used; unused time is returned."}
} - {tab === 'runs' &&

Your evaluations

{!jobs.length &&

Your runs will appear here, with rankings and individual judgments.

}{jobs.map(job => { const open = openRuns[job.id] ?? isActive(job.state); const body = `run-body-${job.id}`; return
+ {tab === 'runs' &&

Your evaluations

{!jobs.length &&

Your runs will appear here, with rankings and individual judgments.

}{jobs.some(j => j.state === 'succeeded') &&
{jobs.filter(j => j.state === 'succeeded').map(job => )}
EvaluationConstitutionModelsScenariosVisibilityResults
{job.name}{job.constitution}{job.models_count}{job.scenario_count}{job.visibility}View results{' · '}Transcripts{' · '}
}{jobs.filter(job => job.state !== 'succeeded' || openRuns[job.id]).map(job => { const open = openRuns[job.id] ?? isActive(job.state); const body = `run-body-${job.id}`; return
{/* Active runs start open; finished ones stay collapsed until clicked */}
{job.state === 'failed' && ['gpu_unavailable', 'allocation_rejected', 'worker_start_timeout'].includes(job.error_code || '') && }{open && setJobs(current=>current.map(j=>j.id===updated.id?updated:j))}/>}{job.publication &&

{job.publication.state === 'published' ? Published in Experiments ↗ : job.publication.state === 'failed' ? job.publication.error : job.publication.state === 'unpublished' ? 'Unlisted from Experiments. Earlier public copies may still exist.' : job.publication.state === 'unpublishing' ? 'Removing public listing…' : 'Publishing to Hugging Face…'}

}
{job.state === 'succeeded' && <>View results}{job.has_artifacts && }
; })}
} diff --git a/next-src/src/components/RunMonitor.tsx b/next-src/src/components/RunMonitor.tsx index 9de1d683..6d40aff9 100644 --- a/next-src/src/components/RunMonitor.tsx +++ b/next-src/src/components/RunMonitor.tsx @@ -3,12 +3,13 @@ import { useEffect, useId, useRef, useState } from 'react'; import { evaluationRequest as request, type EvaluationJob } from '@/lib/evaluation'; type Activity = {text:string;provider?:{events?:{text:string;time:string}[];available:boolean;updated_at?:number;checked_at?:number};pod_name?:string;gpu?:string;progress?:EvaluationJob['progress']}; -const steps=['Queue','GPU startup','Worker ready','Responses & judgments','Rankings','Saving','Complete']; +const defaultSteps=['Queue','GPU startup','Worker ready','Responses & judgments','Rankings','Saving','Complete']; const descriptions=['Waiting for an available execution slot.','RunPod allocates the GPU, downloads the image, and starts its container.','The worker connects and prepares the evaluation.','Models produce responses and judges evaluate them.','EigenBench computes rankings and uncertainty intervals.','Results and transcripts are saved to your account.','The evaluation is finished.']; export const isActive=(state:string)=>['queued','provisioning','running'].includes(state); function duration(seconds:number){return `${Math.floor(seconds/60)}m ${seconds%60}s`;} export function RunMonitor({job,admin=false,onUpdate}:{job:EvaluationJob;admin?:boolean;onUpdate?:(job:EvaluationJob)=>void}){ + const steps = defaultSteps.map(s => job.compute_type === 'cpu' ? s.replace('GPU', 'CPU') : s); const [activity,setActivity]=useState(null),[open,setOpen]=useState(false),[source,setSource]=useState<'worker'|'provider'>('worker'); const [error,setError]=useState(''),[busy,setBusy]=useState(false),[confirm,setConfirm]=useState(false),[follow,setFollow]=useState(true),[selected,setSelected]=useState(null),[now,setNow]=useState(Date.now()); const dialog=useRef(null),output=useRef(null);const id=useId(); @@ -39,6 +40,6 @@ export function RunMonitor({job,admin=false,onUpdate}:{job:EvaluationJob;admin?:
{active&&}
{error&&

{error}

} {open&&
{log||(source==='provider'?'No RunPod startup logs recorded yet.':'Waiting for worker output…')}
{source==='worker'?'Latest 64 KB of worker output':'Recent RunPod system events'}{activity?.pod_name&&` · ${activity.pod_name}`}
} - {e.preventDefault();if(!busy)setConfirm(false);}}>

Cancel this evaluation?

Stop {job.name} and release its GPU. This run cannot resume automatically.

{error&&

{error}

}
 + {e.preventDefault();if(!busy)setConfirm(false);}}>

Cancel this evaluation?

Stop {job.name} and release its compute instance. This run cannot resume automatically.

{error&&

{error}

}
; } diff --git a/next-src/src/lib/evaluation.ts b/next-src/src/lib/evaluation.ts index dc57627b..06d43224 100644 --- a/next-src/src/lib/evaluation.ts +++ b/next-src/src/lib/evaluation.ts @@ -21,6 +21,7 @@ export async function evaluationRequest(path: string, options: RequestInit = {}, return response; } export type EvaluationJob = { + compute_type?: 'gpu' | 'cpu'; omitted_count?: number; publication?: {state: string; slug?: string; url?: string; error?: string} | null; progress?: {title:string;detail:string;step:number;checked_at:number;elapsed_seconds:number;worker_last_seen_at:number|null;gpu_allocated:boolean;cleanup_pending:boolean}; diff --git a/next-src/src/lib/hf.ts b/next-src/src/lib/hf.ts index 9eafe4c4..6c0cbe28 100644 --- a/next-src/src/lib/hf.ts +++ b/next-src/src/lib/hf.ts @@ -3,6 +3,7 @@ // - sessionStorage cache with per-path TTLs // - identical URL shape (HF dataset resolve/main paths) +import { privateRun, privateAssetURL, fetchPrivateJSON } from './run-source'; import { HF_BASE } from './config'; import type { IndexJson, MetaJson, Summary } from './types'; @@ -46,6 +47,7 @@ function ssSet(path: string, data: unknown): void { } export async function hfFetch(path: string): Promise { + if (privateRun(path)) return fetchPrivateJSON(path); if (path in memCache) return memCache[path] as T; const cached = ssGet(path); if (cached !== null) { @@ -62,7 +64,7 @@ export async function hfFetch(path: string): Promise { } export function hfImageURL(path: string): string { - return `${HF_BASE}/${path}`; + return privateAssetURL(path) ?? `${HF_BASE}/${path}`; } export function fetchIndex(): Promise { diff --git a/next-src/src/lib/run-source.ts b/next-src/src/lib/run-source.ts new file mode 100644 index 00000000..b40fff51 --- /dev/null +++ b/next-src/src/lib/run-source.ts @@ -0,0 +1,53 @@ +import { evaluationAuth, evaluationRequest } from './evaluation'; + +type Manifest = { files: Record; expires_in: number }; +const manifests = new Map(); +const pending = new Map>(); +const signedSources = new Map(); + +export function privateRun(path: string): { id: string; file: string } | null { + const m = /^runs\/account\/([a-f0-9-]{36})\/(.+)$/.exec(path); + return m ? { id: m[1], file: m[2] } : null; +} + +export async function preparePrivateRun(id: string): Promise { + const session = (await evaluationAuth()?.auth.getSession())?.data.session; + if (!session) throw new Error('Log in to view your evaluation results.'); + const user = session.user.id, key = `${user}/${id}`; + const saved = manifests.get(id); + if (saved?.user === user && saved.until > Date.now()) return saved.data; + if (pending.has(key)) return pending.get(key)!; + const task = evaluationRequest(`/results/${id}/viewer`).then(r => r.json()).then((data: Manifest) => { + manifests.set(id, { user, until: Date.now() + Math.max(0, data.expires_in - 60) * 1000, data }); + for (const [file, url] of Object.entries(data.files)) signedSources.set(url, `runs/account/${id}/${file}`); + return data; + }).finally(() => pending.delete(key)); + pending.set(key, task); + return task; +} + +export function privateAssetURL(path: string): string | null { + const match = privateRun(path); + if (!match) return null; + return manifests.get(match.id)?.data.files[match.file] || ''; +} + +export async function fetchRunAsset(url: string, init?: RequestInit): Promise { + const path = signedSources.get(url); + if (!path) return fetch(url, init); + const match = privateRun(path)!; + const manifest = await preparePrivateRun(match.id); + const nextURL = manifest.files[match.file]; + if (!nextURL) throw new Error('Result file not found.'); + return fetch(nextURL, init); +} + +export async function fetchPrivateJSON(path: string): Promise { + const match = privateRun(path)!; + const data = await preparePrivateRun(match.id); + const url = data.files[match.file]; + if (!url) throw new Error('Result file not found.'); + const response = await fetch(url, { cache: 'no-store' }); + if (!response.ok) throw new Error('Could not load result file.'); + return response.json(); +} diff --git a/next-src/src/lib/transcripts.ts b/next-src/src/lib/transcripts.ts index 094e83dc..bac9a28e 100644 --- a/next-src/src/lib/transcripts.ts +++ b/next-src/src/lib/transcripts.ts @@ -11,7 +11,8 @@ // HF honours Range requests on this path — 206, uncompressed, true offsets, // CORS-exposed — which is what makes both the windowing and the re-read work. -import { HF_BASE } from './config'; +import { hfImageURL } from './hf'; +import { fetchRunAsset } from './run-source'; /** First window. Small, so the first rows appear without waiting on the file. */ export const FIRST_WINDOW_BYTES = 256 * 1024; @@ -25,7 +26,7 @@ const MAX_WINDOW_BYTES = 8 * 1024 * 1024; const NEWLINE = 0x0a; export function evaluationsURL(slug: string): string { - return `${HF_BASE}/runs/${slug}/evaluations.jsonl`; + return hfImageURL(`runs/${slug}/evaluations.jsonl`); } // ── Reader ────────────────────────────────────────────────────────────────── @@ -126,7 +127,7 @@ export class JsonlWindowReader { /** Fetch one window and return the complete lines it closed out. */ private async readWindow(size: number): Promise { const end = this.offset + size - 1; - const res = await fetch(this.url, { headers: { Range: `bytes=${this.offset}-${end}` } }); + const res = await fetchRunAsset(this.url, { headers: { Range: `bytes=${this.offset}-${end}` } }); // Past the end of the file: nothing left but whatever the carry holds. if (res.status === 416) return this.finish(); @@ -235,7 +236,7 @@ export async function fetchRawRecord( byteStart: number, byteLen: number ): Promise { - const res = await fetch(url, { + const res = await fetchRunAsset(url, { headers: { Range: `bytes=${byteStart}-${byteStart + byteLen - 1}` }, }); if (!res.ok && res.status !== 206) {