Show GPU capacity before submission and explain allocation failures

backend/app/progress.py

@@ -20,6 +20,8 @@ def progress(job, now=None):
     now = int(time.time()) if now is None else now
     state = job['state']; stage = job['stage']
     error = job.get('error_code') or ''
+    config = job.get('config', {})
+    gpu_label = f"{config.get('gpu_count', 1)} × {config.get('gpu_type', 'selected GPU')}"
     provider_error = error.startswith('provider_create_')
     provider_detail = ('RunPod returned HTTP '+error.removeprefix('provider_create_http_')+'. ' if error.startswith('provider_create_http_') else 'The RunPod allocation request did not return a confirmed result. ')
     provider_detail += 'No GPU allocation has been confirmed. The scheduler checks for a late-created instance instead of sending another request.'
@@ -29,7 +31,7 @@ def progress(job, now=None):
         allocation = job.get('allocation') or {}
         if allocation.get('outcome') == 'unavailable' and not job.get('pod_id'):
             delay = max(0, (allocation.get('next_retry_at') or now) - now)
-            title, detail, step = 'Waiting for GPU availability', f"Attempt {allocation['attempts']}: selected GPU unavailable. Next retry in {delay} seconds; retrying for up to 5 minutes.", 1
+            title, detail, step = 'Waiting for GPU availability', f"RunPod has no matching capacity for {gpu_label}. Attempt {allocation['attempts']}. Next retry in {delay} seconds, for up to 5 minutes. No GPU has been allocated and model evaluation has not started. You can cancel and choose another GPU.", 1
         elif provider_error and not job.get('pod_id'):
             title, detail, step = 'GPU allocation not confirmed', provider_detail, 1
         elif job.get('pod_id'):
@@ -47,6 +49,9 @@ def progress(job, now=None):
         title, detail, step = 'Completed', 'Rankings and transcripts are ready.', 6
     else:
         title, detail, step = ('Cancelled' if state == 'cancelled' else 'Failed'), (provider_detail if provider_error else ERRORS.get(job.get('error_code'), 'The evaluation stopped. See worker output for details.')), -1
+    if state == 'failed' and error == 'gpu_unavailable':
+        title = 'GPU capacity unavailable'
+        detail = f'RunPod could not allocate {gpu_label} within 5 minutes. Model evaluation never started. Retry with another GPU or try again when capacity returns.'
     if job.get('config', {}).get('compute_type') == 'cpu':
         title = title.replace('GPU', 'CPU')
         detail = detail.replace('GPU', 'CPU')

backend/app/runpod.py

@@ -31,7 +31,7 @@ def gpu_availability(disk_gb=100, gpu_count=1):
     rows = []
     for gpu in body['data']['gpuTypes']:
         price = gpu.get('lowestPrice') or {}
-        stock = price.get('stockStatus')
+        stock = price.get('stockStatus') if gpu.get('lowestPrice') is not None else 'None'
         rows.append({'id': gpu['id'], 'stock': stock if stock in ('High','Medium','Low','None') else 'Unknown',
                      'price_per_hour': price.get('uninterruptablePrice')})
     return {'gpus': rows, 'checked_at': int(time.time()), 'min_cuda_version': MIN_CUDA_VERSION,

backend/tests/test_backend.py

@@ -332,6 +332,8 @@ def unavailable(job): raise GPUUnavailable()
     assert view['progress']['title'] == 'Waiting for GPU availability'
     assert 'Attempt 1' in view['progress']['detail']
     assert 'Next retry' in view['progress']['detail']
+    assert 'NVIDIA A40' in view['progress']['detail']
+    assert 'model evaluation has not started' in view['progress']['detail']
 
 
 def test_compute_limits_and_cpu_local_models(service, monkeypatch):

backend/tests/test_runpod.py

@@ -21,18 +21,19 @@ def handle(request):
         assert pods.create({'id':str(uuid4()),'config':{}})=='mock-pod'
 
 
-def test_stock_check_preserves_unknown_and_filters_cuda_disk(monkeypatch):
+def test_stock_check_reports_missing_offers_and_filters_cuda_disk(monkeypatch):
     from app.runpod import gpu_availability
     def post(url, json, timeout):
         assert json['variables']['input'] == {'gpuCount': 1, 'secureCloud': True, 'minCudaVersion': '13.0', 'minDisk': 200}
         return httpx.Response(200, request=httpx.Request('POST',url), json={'data':{'gpuTypes':[
             {'id':'A40','lowestPrice':{'stockStatus':'Low','uninterruptablePrice':0.49}},
             {'id':'A100','lowestPrice':None},
-            {'id':'H100','lowestPrice':{'stockStatus':'None','uninterruptablePrice':None}}
+            {'id':'H100','lowestPrice':{'stockStatus':'None','uninterruptablePrice':None}},
+            {'id':'unknown','lowestPrice':{'stockStatus':'unexpected','uninterruptablePrice':None}}
         ]}})
     monkeypatch.setattr('app.runpod.httpx.post',post)
     result=gpu_availability(200)
-    assert [g['stock'] for g in result['gpus']] == ['Low','Unknown','None']
+    assert [g['stock'] for g in result['gpus']] == ['Low','None','None','Unknown']
     assert result['disk_gb']==200
 
 
@@ -86,3 +87,18 @@ def handle(request):
     with httpx.Client(transport=httpx.MockTransport(handle)) as client:
         pods.client.close(); pods.client=client
         assert pods.create({'id':str(uuid4()),'config':{'gpu_count':4,'volume_gb':200}})=='multi-pod'
+
+
+def test_every_selectable_gpu_is_forwarded_without_substitution():
+    from typing import get_args
+    from app.models import EvaluationRequest
+    for gpu in get_args(EvaluationRequest.model_fields['gpu_type'].annotation):
+        pods=RunPod(Settings(environment='test',worker_secret='x'*40))
+        def handle(request):
+            payload=json.loads(request.content)['variables']['input']
+            assert payload['gpuTypeId']==gpu
+            assert payload['gpuCount']==1
+            return httpx.Response(200,json={'data':{'podFindAndDeployOnDemand':{'id':'test-pod'}}})
+        with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+            pods.client.close();pods.client=client
+            assert pods.create({'id':str(uuid4()),'config':{'gpu_type':gpu,'gpu_count':1}})=='test-pod'

next-src/src/app/pages.css

@@ -1005,3 +1005,6 @@ body:has(.arena-home) .page-hills {
     transition: none;
   }
 }
+
+.eval-capacity-note { padding: 12px 16px; border: 1px solid var(--border); border-radius: 12px; font-size: 13px; line-height: 1.6; background: var(--bg-elevated); }
+.eval-capacity-note[data-warning='true'] { border-color: #af874b; border-left-width: 3px; }

next-src/src/components/EvaluationRunner.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { useEffect, useRef, useState } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
 import type { Session } from '@supabase/supabase-js';
 import { evaluationAPI, evaluationAuth, evaluationRequest as request, type EvaluationJob as Job } from '@/lib/evaluation';
 import { CONSTITUTIONS_DATA } from '@/lib/constitutions-data';
@@ -40,7 +40,8 @@ export function EvaluationRunner() {
   const [computeType, setComputeType] = useState('gpu'); const [gpuCount, setGPUCount] = useState(1);
   const [cpuCount, setCPUCount] = useState(4); const [cpuFlavor, setCPUFlavor] = useState('cpu3g'); const [volume, setVolume] = useState(0);
   const [gpu, setGPU] = useState(gpuTypes[0]); const [disk, setDisk] = useState(100);
-  const [stock, setStock] = useState<{ gpus: { id: string; stock: string; price_per_hour: number | null }[]; checked_at: number; disk_gb: number; gpu_count: number } | null>(null);
+  const [stock, setStock] = useState<{ gpus: { id: string; stock: string; price_per_hour: number | null }[]; checked_at: number; disk_gb: number; gpu_count: number; min_cuda_version: string } | null>(null);
+  const stockRequest = useRef(0);
   const [stockBusy, setStockBusy] = useState(false); const [stockError, setStockError] = useState('');
   const [visibility, setVisibility] = useState('private');
   const [accessError, setAccessError] = useState('');
@@ -150,14 +151,29 @@ export function EvaluationRunner() {
     } catch (e) { setError((e as Error).message); }
   }
 
-  async function checkStock() {
+  const effectiveDisk = ownKeys || isAdmin ? disk + volume : 100;
+  const effectiveGPUCount = ownKeys || isAdmin ? gpuCount : 1;
+  const effectiveGPU = ownKeys || isAdmin ? gpu : gpuTypes[0];
+  const usesGPU = !(ownKeys || isAdmin) || computeType === 'gpu';
+  const checkStock = useCallback(async () => {
+    const sequence = ++stockRequest.current;
     setStockBusy(true); setStockError(''); setStock(null);
     try {
-      const response = await request(`/compute/availability?disk_gb=${ownKeys || isAdmin ? disk + volume : 100}&gpu_count=${ownKeys || isAdmin ? gpuCount : 1}`);
-      setStock(await response.json());
-    } catch (e) { setStockError((e as Error).message); }
-    finally { setStockBusy(false); }
-  }
+      const response = await request(`/compute/availability?disk_gb=${effectiveDisk}&gpu_count=${effectiveGPUCount}`);
+      const data = await response.json();
+      if (sequence === stockRequest.current) setStock(data);
+    } catch (e) { if (sequence === stockRequest.current) setStockError((e as Error).message); }
+    finally { if (sequence === stockRequest.current) setStockBusy(false); }
+  }, [effectiveDisk, effectiveGPUCount]);
+  useEffect(() => {
+    if (access !== 'approved' || !usesGPU || tab !== 'new') return;
+    const debounce = setTimeout(() => void checkStock(), 400);
+    const timer = setInterval(() => void checkStock(), 60000);
+    return () => { clearTimeout(debounce); clearInterval(timer); ++stockRequest.current; setStockBusy(false); };
+  }, [access, usesGPU, tab, session?.user.id, checkStock]);
+  const matchingStock = stock && stock.disk_gb === effectiveDisk && stock.gpu_count === effectiveGPUCount ? stock : null;
+  const selectedStock = matchingStock?.gpus.find(item => item.id === effectiveGPU);
+  const capacityMessage = stockBusy ? 'Checking matching RunPod capacity…' : stockError ? 'Availability could not be checked. Allocation may still fail; refresh or submit to try.' : selectedStock?.stock === 'None' ? 'No matching capacity reported. Choose another GPU, or submit to retry allocation for up to 5 minutes. Evaluation starts only after a GPU is allocated.' : selectedStock?.stock === 'Low' ? 'Limited capacity. Allocation can fail even when stock is reported; we retry for up to 5 minutes.' : !selectedStock || selectedStock.stock === 'Unknown' ? 'Capacity is unconfirmed. RunPod has not reported matching stock; submitting will attempt allocation.' : 'Capacity reported available. This is not a reservation; RunPod confirms availability when allocating your instance.';
 
   const collectionOptions = (advancedOptions.collection || {}) as {generation?:Record<string,{max_tokens?:number}>;failure_policy?:string};
   function setBudget(phase:string,value:string) {
@@ -320,9 +336,9 @@ export function EvaluationRunner() {
             </>}
           </fieldset>
           {computeType === 'gpu' && <>
-          <div className="eval-gpu-head"><span className="eval-gpu-label">GPU</span><button type="button" className="eval-link-button" disabled={stockBusy} onClick={checkStock}>{stockBusy ? 'Checking RunPod…' : 'Check availability'}</button></div>
-          {stockError && <p className="eval-help" role="status">{stockError}</p>}
-          <fieldset className="eval-gpu" disabled={!ownKeys && !isAdmin}><legend className="sr-only">GPU</legend><div className="eval-gpu-grid" aria-live="polite">{gpuTypes.map(g => { const item = stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) ? stock.gpus.find(x => x.id === g) : undefined; return <label className="eval-choice-card eval-gpu-card" key={g} data-stock={item ? (item.stock === 'None' ? 'none' : item.stock === 'Unknown' ? 'unknown' : 'ok') : undefined}><input type="radio" name="gpu" checked={(ownKeys || isAdmin ? gpu : gpuTypes[0]) === g} onChange={() => setGPU(g)} /><strong>{g.replace('NVIDIA ', '').replace('GeForce ', '')}</strong><small>{item ? <>{item.stock === 'None' ? 'Unavailable' : item.stock === 'Unknown' ? 'Not reported' : `${item.stock} stock`}{item.price_per_hour != null && ` · $${item.price_per_hour.toFixed(2)}/hr`}</> : 'NVIDIA'}</small></label>; })}</div><small>GPUs are allocated together on one instance.{!ownKeys && !isAdmin && ' Lab-funded runs use the default GPU.'}{stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) && ` Secure Cloud · CUDA 13+ · ${stock.disk_gb} GB disk · checked ${new Date(stock.checked_at * 1000).toLocaleTimeString()}. Stock can change before allocation.`}</small></fieldset>
+          <div className="eval-gpu-head"><span className="eval-gpu-label">GPU</span><button type="button" className="eval-link-button" disabled={stockBusy} onClick={() => void checkStock()}>{stockBusy ? 'Checking RunPod…' : 'Check availability'}</button></div>
+          <p className="eval-capacity-note" data-warning={selectedStock?.stock === 'None' || selectedStock?.stock === 'Low' || !!stockError} role="status">{capacityMessage}</p>
+          <fieldset className="eval-gpu" disabled={!ownKeys && !isAdmin}><legend className="sr-only">GPU</legend><div className="eval-gpu-grid" aria-live="polite">{gpuTypes.map(g => { const item = stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) ? stock.gpus.find(x => x.id === g) : undefined; return <label className="eval-choice-card eval-gpu-card" key={g} data-stock={item ? (item.stock === 'None' ? 'none' : item.stock === 'Unknown' ? 'unknown' : 'ok') : undefined}><input type="radio" name="gpu" checked={(ownKeys || isAdmin ? gpu : gpuTypes[0]) === g} onChange={() => setGPU(g)} /><strong>{g.replace('NVIDIA ', '').replace('GeForce ', '')}</strong><small>{item ? <>{item.stock === 'None' ? 'No matching capacity' : item.stock === 'Unknown' ? 'Capacity unconfirmed' : `${item.stock} stock`}{item.price_per_hour != null && ` · $${item.price_per_hour.toFixed(2)}/hr`}</> : stockBusy ? 'Checking capacity…' : 'Capacity not checked'}</small></label>; })}</div><small>GPUs are allocated together on one instance. Model size must fit the selected GPU memory.{!ownKeys && !isAdmin && ' Lab-funded runs use the default GPU.'}{stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) && ` Secure Cloud · CUDA ${stock.min_cuda_version}+ · ${stock.disk_gb} GB disk · checked ${new Date(stock.checked_at * 1000).toLocaleTimeString()}. Stock can change before allocation.`}</small></fieldset>
           </>}
           <div className="eval-fields"><label>Container storage (GB)<input disabled={!ownKeys && !isAdmin} type="number" min={50} max={Math.min(limits.max_disk_gb ?? 1000,1000)} step={10} value={ownKeys || isAdmin ? disk : 100} onChange={e => setDisk(Number(e.target.value))} /><button type="button" disabled={!ownKeys && !isAdmin} onClick={()=>setDisk(Math.min(limits.max_disk_gb ?? 1000,1000))}>Max allowed</button><small>Model cache and working files. Released after results are saved.</small></label><label>Workspace volume (GB)<input type="number" disabled={!ownKeys && !isAdmin} min={0} max={Math.min(limits.max_disk_gb ?? 1000,1000)} step={10} value={ownKeys || isAdmin ? volume : 0} onChange={e => setVolume(Number(e.target.value))}/><small>Optional /workspace disk. Survives pod restarts; deleted when the pod is terminated. Results are saved separately.</small></label><label>Results<select value={visibility} onChange={e => setVisibility(e.target.value)}><option value="private">Private · only in my account</option><option disabled={!limits.allow_public_results} value="public">Public · list in Experiments</option></select></label></div>
           {visibility === 'public' && <small>Completed rankings, scenarios, responses, and judgments will be visible to everyone. Public results are copied to Hugging Face. Unlisting removes them from the current listing, but repository history and downloaded copies remain public.</small>}