Add compute options and reuse experiment views for private results

.github/workflows/backend-tests.yml

@@ -31,7 +31,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           repository: jchang153/EigenBench
-          ref: 5e7520f71fd74fb3c884ae1a02534c8489348dbd
+          ref: c4ef51394d30ad9824eb76012db24e813f1a76c1
           path: upstream-eigenbench
       - uses: actions/setup-python@v5
         with:

backend/Dockerfile.worker

@@ -1,6 +1,6 @@
 # Reuse the verified CUDA toolkit and GPU dependency image.
 FROM ghcr.io/valuearena/evaluation-worker@sha256:66adcf3c9f6672826eba35cab072e8d5e5db3cbe67fcf2c71dac5a76ae4d533d
-ARG EIGENBENCH_REVISION=5e7520f71fd74fb3c884ae1a02534c8489348dbd
+ARG EIGENBENCH_REVISION=c4ef51394d30ad9824eb76012db24e813f1a76c1
 RUN cd /opt/eigenbench \
     && git remote set-url origin https://github.com/jchang153/EigenBench.git \
     && git fetch origin "$EIGENBENCH_REVISION" \

backend/Dockerfile.worker-base

@@ -4,7 +4,7 @@ FROM nvidia/cuda:13.0.2-devel-ubuntu22.04@sha256:6b6617592b94e7dcc6ffbe6d00720ee
 FROM python:3.11-slim
 COPY --from=cuda-toolkit /usr/local/cuda-13.0 /usr/local/cuda
 ENV CUDA_HOME=/usr/local/cuda PATH=/usr/local/cuda/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda/lib64
-ARG EIGENBENCH_REVISION=5e7520f71fd74fb3c884ae1a02534c8489348dbd
+ARG EIGENBENCH_REVISION=c4ef51394d30ad9824eb76012db24e813f1a76c1
 RUN apt-get update && apt-get install -y --no-install-recommends git libgomp1 build-essential && rm -rf /var/lib/apt/lists/*
 RUN git clone https://github.com/jchang153/EigenBench.git /opt/eigenbench \
     && cd /opt/eigenbench && git checkout "$EIGENBENCH_REVISION"

backend/README.md

@@ -433,3 +433,21 @@ service HF_TOKEN when no personal token is supplied. Configure that service toke
 on both API and scheduler; own-provider runs never inherit the service token.
 Native OpenRouter attempts emit `[API]` lines in worker logs with timing, token
 usage and retry status, without prompts or response bodies.
+
+### Compute selection
+
+Evaluations can request `compute_type: "cpu"` for API-only panels, with
+`cpu_count` (2, 4, 8, 16, or 32) and `cpu_flavor` (`cpu3c` for 2 GB RAM per
+vCPU or `cpu3g` for 4 GB). Hugging Face panels require GPU compute.
+Native GPU runs accept `gpu_count` of 1, 2, 4, or 8; all GPUs share one
+instance and each local model uses tensor parallelism across that count.
+Inspect currently accepts one GPU. Models must support the chosen parallel size.
+
+`disk_gb` controls container storage. Optional `volume_gb` mounts workspace
+storage at `/workspace`; it survives restarts but is deleted with the pod.
+Saved results remain in the configured results storage. The admin disk limit
+applies to the sum of both disks; CPU and GPU counts have separate admin limits.
+Custom compute is available to admins and users supplying their own provider keys.
+
+The worker image must include the pinned EigenBench revision for multi-GPU
+execution; `EIGENBENCH_TENSOR_PARALLEL_SIZE` is set by the scheduler.

backend/app/api.py

@@ -28,7 +28,7 @@
 def public_job(job):
     return {key: job[key] for key in ('id', 'state', 'stage', 'created_at', 'started_at', 'finished_at',
             'reserved_credits', 'charged_credits', 'error_code')} | {
-                'progress': progress(job), 'name': job['config']['name'], 'engine': job['config']['engine'],
+                'compute_type':job['config'].get('compute_type','gpu'), 'progress': progress(job), 'name': job['config']['name'], 'engine': job['config']['engine'],
                 'has_artifacts': bool(job['artifact']),
                 'funding': job['config'].get('funding', 'service'),
                 'visibility': job['config'].get('visibility', 'private'),
@@ -154,20 +154,20 @@ def health(): return {'status': 'ok'}
 
     @app.get('/models')
     def models():
-        return {'models': [{'id': key, 'label': entry['label']} for key, entry in catalog.items()],
+        return {'models': [{'id': key, 'label': entry['label'], 'provider': 'huggingface' if isinstance(entry['ref'], dict) else 'openrouter'} for key, entry in catalog.items()],
                 'engines': ['native', 'inspect'], 'default_engine': 'native', 'evaluation_mode': 'direct_rating'}
 
     @app.get('/models/openrouter')
     def available_openrouter(user_id=Depends(user)):
         return {'models': openrouter_models()}
 
     @app.get('/compute/availability')
-    def compute_availability(disk_gb: int = Query(default=100, ge=50, le=1000), user_id=Depends(user)):
+    def compute_availability(disk_gb: int = Query(default=100, ge=50, le=2000), gpu_count: int = Query(default=1, ge=1, le=8), user_id=Depends(user)):
         if db.member(user_id)['status'] != 'approved':
             raise HTTPException(403, 'Account approval required')
         from .runpod import gpu_availability
         try:
-            return gpu_availability(disk_gb)
+            return gpu_availability(disk_gb, gpu_count)
         except Exception:
             raise HTTPException(503, 'RunPod availability could not be checked. Try again shortly.') from None
 
@@ -186,6 +186,8 @@ def spec_preview(incoming: EvaluationRequest, user_id=Depends(user)):
         import pprint
         config = effective_config(incoming,user_id)
         config['model_refs'] = resolve_models(incoming, catalog, incoming.hf_token.get_secret_value() or (cfg.hf_token if incoming.funding == 'service' else ''))
+        if incoming.compute_type == 'cpu' and any(isinstance(ref, dict) for ref in config['model_refs'].values()):
+            raise HTTPException(422, 'CPU evaluations support API models only; Hugging Face models require GPU')
         spec = build_spec(config, '.')
         return {'spec': spec, 'python': 'RUN_SPEC = ' + pprint.pformat(spec, sort_dicts=False) + '\n'}
 
@@ -219,10 +221,12 @@ async def submit(request: Request, user_id=Depends(user), idempotency_key: str =
             raise HTTPException(422, 'Supply both your OpenRouter and RunPod API keys')
         if incoming.funding == 'own_keys':
             await run_in_threadpool(verify_provider_keys, keys)
-        if incoming.funding == 'service' and db.member(user_id)['role']!='admin' and (incoming.gpu_type != cfg.runpod_gpu_type or incoming.disk_gb != cfg.runpod_disk_gb):
+        if incoming.funding == 'service' and db.member(user_id)['role']!='admin' and (incoming.gpu_type != cfg.runpod_gpu_type or incoming.disk_gb != cfg.runpod_disk_gb or incoming.gpu_count != 1 or incoming.compute_type != 'gpu' or incoming.cpu_count != 4 or incoming.cpu_flavor != 'cpu3g' or incoming.volume_gb != 0):
             raise HTTPException(422, 'Custom compute requires your own provider keys')
         digest = hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest()
         # Snapshot the catalog: queued jobs do not silently change if administrators update it.
+        if incoming.compute_type == 'cpu' and any(isinstance(ref, dict) for ref in refs.values()):
+            raise HTTPException(422, 'CPU evaluations support API models only. Select GPU for Hugging Face models.')
         config['model_refs'] = refs
         if incoming.funding != 'own_keys': keys = {}
         if incoming.hf_token.get_secret_value(): keys['hf_token'] = incoming.hf_token.get_secret_value()
@@ -253,6 +257,18 @@ def results(job_id: UUID, authorization: str | None = Header(default=None)):
         if not result: raise HTTPException(404, 'Results are not available yet')
         return {'job': present(job), 'criteria': job['config']['criteria'][:job['config'].get('advanced_spec', {}).get('constitution', {}).get('num_criteria')], **result}
 
+    @app.get('/results/{job_id}/viewer')
+    def result_viewer(job_id: UUID, authorization: str | None = Header(default=None)):
+        job = readable(job_id, authorization)
+        if job['state'] != 'succeeded': raise HTTPException(404, 'Results are not available yet')
+        from .viewer import viewer_manifest
+        if isinstance(storage, LocalStorage):
+            raise HTTPException(503, 'The full viewer requires signed result storage')
+        try:
+            return viewer_manifest(db, storage, job, cfg.max_artifact_bytes)
+        except Exception:
+            raise HTTPException(503, 'Result files could not be prepared. Please retry shortly.') from None
+
     @app.get('/results/{job_id}/records/{batch}')
     def records(job_id: UUID, batch: int, authorization: str | None = Header(default=None)):
         job = readable(job_id, authorization)
@@ -274,7 +290,7 @@ def activity(job):
         return {'text':db.get_presentation(job['id'],'log') or '',
                 'provider':db.get_presentation(job['id'],'provider_activity'),
                 'pod_name':'valuearena-'+job['id'] if job.get('pod_id') else None,
-                'gpu':job['config'].get('gpu_type',cfg.runpod_gpu_type),
+                'gpu':f"{job['config'].get('cpu_count', 4)} vCPUs" if job['config'].get('compute_type') == 'cpu' else f"{job['config'].get('gpu_count', 1)} × {job['config'].get('gpu_type',cfg.runpod_gpu_type)}",
                 'state':job['state'],'stage':job['stage'],'progress':progress({**job, 'allocation':db.get_presentation(job['id'],'allocation')})}
 
     @app.get('/evaluations/{job_id}/logs')

backend/app/config.py

@@ -20,7 +20,7 @@ class Settings(BaseSettings):
     runpod_api_key: str = ''
     worker_image: str = ''
     runpod_gpu_type: str = 'NVIDIA A40'
-    runpod_gpu_count: int = Field(default=1, ge=1, le=1)
+    runpod_gpu_count: int = Field(default=1, ge=1, le=8)
     runpod_disk_gb: int = Field(default=100, ge=30, le=1000)
     max_running_jobs: int = Field(default=1, ge=1, le=10)
     scheduler_interval: int = Field(default=10, ge=1, le=60)

backend/app/governance.py

@@ -15,6 +15,8 @@ class Limits(BaseModel):
     max_tokens: int | None = Field(None, ge=128)
     max_outstanding_jobs: int | None = Field(None, ge=1)
     max_disk_gb: int | None = Field(None, ge=50)
+    max_gpu_count: int | None = Field(None, ge=1)
+    max_cpu_count: int | None = Field(None, ge=2)
     max_workers: int | None = Field(None, ge=1)
     max_bootstraps: int | None = Field(None, ge=1)
     require_credits: bool = False
@@ -73,7 +75,9 @@ def enforce(config, limits):
     checks = [(len(config['models']), 'max_models'),
         (len(config['scenarios']) or config.get('scenario_count',200), 'max_scenarios'),
         (config['max_runtime_seconds'],'max_runtime_seconds'),
-        (config['disk_gb'],'max_disk_gb')]
+        (config['disk_gb'] + config.get('volume_gb', 0),'max_disk_gb'),
+        (config.get('gpu_count', 1) if config.get('compute_type', 'gpu') == 'gpu' else 0, 'max_gpu_count'),
+        (config.get('cpu_count', 4) if config.get('compute_type') == 'cpu' else 0, 'max_cpu_count')]
     advanced = AdvancedSpec.model_validate(config.get('advanced_spec',{}))
     if advanced.dataset.count: checks.append((advanced.dataset.count,'max_scenarios'))
     generation = advanced.collection.generation.model_dump()

backend/app/models.py

@@ -35,6 +35,11 @@ class EvaluationRequest(BaseModel):
     openrouter_key: SecretStr = Field(default=SecretStr(''), max_length=512)
     hf_token: SecretStr = Field(default=SecretStr(''), max_length=512, exclude=True)
     runpod_key: SecretStr = Field(default=SecretStr(''), max_length=512)
+    compute_type: Literal['gpu', 'cpu'] = 'gpu'
+    gpu_count: Literal[1, 2, 4, 8] = 1
+    cpu_count: Literal[2, 4, 8, 16, 32] = 4
+    cpu_flavor: Literal['cpu3c', 'cpu3g'] = 'cpu3g'
+    volume_gb: int = Field(default=0, ge=0, le=1000)
     gpu_type: Literal['NVIDIA A40', 'NVIDIA RTX A6000', 'NVIDIA GeForce RTX 4090', 'NVIDIA A100 80GB PCIe', 'NVIDIA H100 80GB HBM3'] = 'NVIDIA A40'
     disk_gb: int = Field(default=100, ge=50, le=1000)
     engine: Literal['native', 'inspect'] = 'native'
@@ -57,6 +62,10 @@ def unique_inputs(self):
             self.scenario_source = 'custom'
         if self.scenario_source == 'custom' and not self.scenarios:
             raise ValueError('Supply at least one custom scenario')
+        if self.compute_type == 'cpu' and self.gpu_count != 1:
+            raise ValueError('CPU evaluations cannot request multiple GPUs')
+        if self.engine == 'inspect' and self.gpu_count > 1:
+            raise ValueError('Multiple GPUs currently require the Native engine')
         ids = [m.id for m in self.custom_models]
         if len(set(ids)) != len(ids) or not set(ids).issubset(self.models):
             raise ValueError('Custom model IDs must be unique and selected')

backend/app/progress.py

@@ -47,6 +47,9 @@ def progress(job, now=None):
         title, detail, step = 'Completed', 'Rankings and transcripts are ready.', 6
     else:
         title, detail, step = ('Cancelled' if state == 'cancelled' else 'Failed'), (provider_detail if provider_error else ERRORS.get(job.get('error_code'), 'The evaluation stopped. See worker output for details.')), -1
+    if job.get('config', {}).get('compute_type') == 'cpu':
+        title = title.replace('GPU', 'CPU')
+        detail = detail.replace('GPU', 'CPU')
     end = job.get('finished_at') or now
     return {'title': title, 'detail': detail, 'step': step, 'checked_at': now,
             'elapsed_seconds': max(0, end-job['created_at']),

backend/app/runpod.py

@@ -14,15 +14,15 @@ class AllocationRejected(RuntimeError):
     pass
 
 
-def gpu_availability(disk_gb=100):
+def gpu_availability(disk_gb=100, gpu_count=1):
     # Public stock lookup: no user/provider credentials are sent or stored.
     query = """query($input: GpuLowestPriceInput!) {
         gpuTypes { id lowestPrice(input: $input) {
             stockStatus uninterruptablePrice
         } }
     }"""
     response = httpx.post('https://api.runpod.io/graphql', json={
-        'query': query, 'variables': {'input': {'gpuCount': 1, 'secureCloud': True,
+        'query': query, 'variables': {'input': {'gpuCount': gpu_count, 'secureCloud': True,
         'minCudaVersion': MIN_CUDA_VERSION, 'minDisk': disk_gb}}}, timeout=15)
     response.raise_for_status()
     body = response.json()
@@ -35,7 +35,7 @@ def gpu_availability(disk_gb=100):
         rows.append({'id': gpu['id'], 'stock': stock if stock in ('High','Medium','Low','None') else 'Unknown',
                      'price_per_hour': price.get('uninterruptablePrice')})
     return {'gpus': rows, 'checked_at': int(time.time()), 'min_cuda_version': MIN_CUDA_VERSION,
-            'disk_gb': disk_gb, 'cloud': 'SECURE'}
+            'disk_gb': disk_gb, 'gpu_count': gpu_count, 'cloud': 'SECURE'}
 
 
 class RunPod:
@@ -57,13 +57,29 @@ def create(self, job):
                'VA_WORKER_TOKEN': worker_token(self.cfg.worker_secret, job['id']),
                'OPENROUTER_API_KEY': self.cfg.openrouter_api_key,
                'HF_TOKEN': self.cfg.hf_token}
+        config = job['config']
+        if config.get('compute_type') == 'cpu':
+            response = self.client.post('/pods', json={
+                'name': self.name(job['id']), 'imageName': self.cfg.worker_image,
+                'computeType': 'CPU', 'cloudType': 'SECURE',
+                'vcpuCount': config.get('cpu_count', 4),
+                'cpuFlavorIds': [config.get('cpu_flavor', 'cpu3g')],
+                'containerDiskInGb': config.get('disk_gb', self.cfg.runpod_disk_gb),
+                'volumeInGb': config.get('volume_gb', 0), 'volumeMountPath': '/workspace',
+                'env': env, 'ports': [],
+            })
+            response.raise_for_status()
+            pod_id = response.json().get('id')
+            if not pod_id: raise RuntimeError('RunPod did not confirm CPU allocation')
+            return pod_id
+        env['EIGENBENCH_TENSOR_PARALLEL_SIZE'] = str(config.get('gpu_count', 1))
         # GraphQL supports a minimum version; the REST enum omits newer CUDA versions.
         payload = {'name': self.name(job['id']), 'imageName': self.cfg.worker_image,
             'cloudType': 'SECURE', 'computeType': 'GPU',
             'gpuTypeId': job['config'].get('gpu_type', self.cfg.runpod_gpu_type),
-            'gpuCount': self.cfg.runpod_gpu_count,
+            'gpuCount': config.get('gpu_count', self.cfg.runpod_gpu_count),
             'containerDiskInGb': job['config'].get('disk_gb', self.cfg.runpod_disk_gb),
-            'minCudaVersion': MIN_CUDA_VERSION, 'volumeInGb': 0, 'ports': '',
+            'minCudaVersion': MIN_CUDA_VERSION, 'volumeInGb': config.get('volume_gb', 0), 'volumeMountPath': '/workspace', 'ports': '',
             'startSsh': False, 'startJupyter': False,
             'env': [{'key': key, 'value': value} for key, value in env.items()]}
         response = self.client.post('https://api.runpod.io/graphql', json={

backend/app/spec.py

@@ -8,6 +8,8 @@
 
 def build_spec(config, directory):
     request = EvaluationRequest.model_validate({k: v for k, v in config.items() if k != 'model_refs'})
+    if request.compute_type == 'cpu' and any(isinstance(ref, dict) for ref in config['model_refs'].values()):
+        raise ValueError('CPU evaluations support API models only; Hugging Face models require GPU')
     root = Path(directory)
     spec = {
         'name': request.name,

backend/app/storage.py

@@ -12,10 +12,10 @@ def __init__(self, settings):
         if not settings.supabase_secret_key.startswith('sb_secret_'):
             self.headers['Authorization'] = 'Bearer ' + settings.supabase_secret_key
 
-    def put(self, key, path):
+    def put(self, key, path, content_type='application/gzip'):
         url = f'{self.root}/object/{self.settings.storage_bucket}/{quote(key, safe="/")}'
         with open(path, 'rb') as source:
-            response = httpx.post(url, headers={**self.headers, 'Content-Type': 'application/gzip', 'x-upsert': 'true'},
+            response = httpx.post(url, headers={**self.headers, 'Content-Type': content_type, 'x-upsert': 'true'},
                                   content=source, timeout=180)
         response.raise_for_status()
 
@@ -42,7 +42,7 @@ class LocalStorage:
     def __init__(self, root):
         self.root = Path(root)
 
-    def put(self, key, path):
+    def put(self, key, path, content_type='application/gzip'):
         import shutil
         dest = self.root / key
         dest.parent.mkdir(parents=True, exist_ok=True)

backend/app/upstream.py

@@ -1,6 +1,6 @@
 """Pinned runner contract. Defaults are checked against upstream source in CI."""
 REPOSITORY = 'https://github.com/jchang153/EigenBench'
-REVISION = '5e7520f71fd74fb3c884ae1a02534c8489348dbd'
+REVISION = 'c4ef51394d30ad9824eb76012db24e813f1a76c1'
 MAX_LORA_RANK = 512
 # Used for preview and limit validation only; omitted settings are resolved by EigenBench.
 GENERATION_DEFAULTS = {

backend/app/viewer.py

@@ -0,0 +1,39 @@
+"""Prepare the published viewer format inside private result storage."""
+import mimetypes
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from threading import Lock
+
+from .publication import export_bundle
+
+_prepare_lock = Lock()
+
+
+def viewer_manifest(db, storage, job, max_bytes):
+    # Existing successful runs are upgraded lazily; no evaluation is re-run.
+    with _prepare_lock:
+        saved = db.get_presentation(job['id'], 'viewer_files')
+        if not saved:
+            summary = db.get_presentation(job['id'], 'summary')
+            if not summary or not job.get('artifact'):
+                raise ValueError('Results are not available yet')
+            with tempfile.TemporaryFile() as archive:
+                storage.fetch(job['artifact'], archive, max_bytes)
+                archive.seek(0)
+                files, _ = export_bundle(job, archive, summary, max_bytes)
+            keys = {name: f"viewer/{job['user_id']}/{job['id']}/{name}" for name in files}
+            with tempfile.TemporaryDirectory() as directory:
+                def upload(item):
+                    name, data = item
+                    path = Path(directory)/name
+                    path.parent.mkdir(parents=True, exist_ok=True)
+                    path.write_bytes(data)
+                    storage.put(keys[name], path, content_type=mimetypes.guess_type(name)[0] or 'application/octet-stream')
+                with ThreadPoolExecutor(max_workers=4) as pool:
+                    list(pool.map(upload, files.items()))
+            saved = {'files': keys}
+            db.put_presentation(job['id'], 'viewer_files', saved)
+    with ThreadPoolExecutor(max_workers=4) as pool:
+        urls = dict(pool.map(lambda item: (item[0], storage.download_url(item[1])), saved['files'].items()))
+    return {'files': urls, 'expires_in': 300}