diff --git a/backend/app/api.py b/backend/app/api.py index df7c8607..896756f7 100644 --- a/backend/app/api.py +++ b/backend/app/api.py @@ -84,9 +84,16 @@ def effective_config(incoming,user_id): except (ValueError,ValidationError): raise HTTPException(422,'Spec overrides conflict with site defaults or model panel') from None config['advanced_spec']=advanced.model_dump(exclude_unset=True,exclude_none=True) - enforce(config,db.effective_limits(user_id)) return config + def select_compute(config,refs,user_id): + local = any(isinstance(ref,dict) for ref in refs.values()) + if local and config['compute_type']=='cpu': + raise HTTPException(422,'CPU evaluations support API models only; Hugging Face models require GPU') + if not local: + config.update(compute_type='cpu',gpu_count=1) + enforce(config,db.effective_limits(user_id)) + @app.get('/admin') def admin_data(actor=Depends(administrator)): return db.admin_snapshot() | {'policy_schema':Policy.model_json_schema(),'limits_schema':Limits.model_json_schema()} @@ -186,8 +193,7 @@ def spec_preview(incoming: EvaluationRequest, user_id=Depends(user)): import pprint config = effective_config(incoming,user_id) config['model_refs'] = resolve_models(incoming, catalog, incoming.hf_token.get_secret_value() or (cfg.hf_token if incoming.funding == 'service' else '')) - if incoming.compute_type == 'cpu' and any(isinstance(ref, dict) for ref in config['model_refs'].values()): - raise HTTPException(422, 'CPU evaluations support API models only; Hugging Face models require GPU') + select_compute(config,config['model_refs'],user_id) spec = build_spec(config, '.') return {'spec': spec, 'python': 'RUN_SPEC = ' + pprint.pformat(spec, sort_dicts=False) + '\n'} @@ -218,12 +224,11 @@ async def submit(request: Request, user_id=Depends(user), idempotency_key: str = raise HTTPException(422, 'Supply both your OpenRouter and RunPod API keys') if incoming.funding == 'own_keys': await run_in_threadpool(verify_provider_keys, keys) - if incoming.funding == 'service' and db.member(user_id)['role']!='admin' and (incoming.gpu_type != cfg.runpod_gpu_type or incoming.disk_gb != cfg.runpod_disk_gb or incoming.gpu_count != 1 or incoming.compute_type != 'gpu' or incoming.cpu_count != 4 or incoming.cpu_flavor != 'cpu3g' or incoming.volume_gb != 0): + if incoming.funding == 'service' and db.member(user_id)['role']!='admin' and (incoming.gpu_type != cfg.runpod_gpu_type or incoming.disk_gb != cfg.runpod_disk_gb or incoming.gpu_count != 1 or incoming.cpu_count != 4 or incoming.cpu_flavor != 'cpu3g' or incoming.volume_gb != 0): raise HTTPException(422, 'Custom compute requires your own provider keys') + select_compute(config,refs,user_id) digest = hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest() # Snapshot the catalog: queued jobs do not silently change if administrators update it. - if incoming.compute_type == 'cpu' and any(isinstance(ref, dict) for ref in refs.values()): - raise HTTPException(422, 'CPU evaluations support API models only. Select GPU for Hugging Face models.') config['model_refs'] = refs if incoming.funding != 'own_keys': keys = {} if incoming.hf_token.get_secret_value(): keys['hf_token'] = incoming.hf_token.get_secret_value() diff --git a/backend/tests/test_backend.py b/backend/tests/test_backend.py index e521825b..78588a52 100644 --- a/backend/tests/test_backend.py +++ b/backend/tests/test_backend.py @@ -320,7 +320,8 @@ def create(self, job): assert db.get(job_id)['error_code'] == 'worker_start_timeout' -def test_unavailable_progress_shows_attempt_and_next_retry(service): +def test_unavailable_progress_shows_attempt_and_next_retry(service, monkeypatch): + monkeypatch.setattr('app.api.resolve_models',lambda *args:{'a':{'provider':'hf_local'},'b':'org/b'}) from app.runpod import GPUUnavailable cfg,db,client=service job_id=submit(client).json()['id'] @@ -348,3 +349,40 @@ def test_compute_limits_and_cpu_local_models(service, monkeypatch): monkeypatch.setattr('app.api.resolve_models', lambda *args: {'a':{'provider':'hf_local'},'b':'org/b'}) result=client.post('/spec-preview',json=payload(compute_type='cpu'),headers={'Authorization':'Bearer '+USER}) assert result.status_code==422 and 'API models only' in result.json()['detail'] + + +@pytest.mark.parametrize('requested_compute',['gpu','cpu']) +def test_api_only_panels_automatically_use_cpu_for_members(service,requested_compute,monkeypatch): + cfg,db,client=service + config=payload(compute_type=requested_compute) + response=submit(client,config) + assert response.status_code==202,response.text + job=db.get(response.json()['id']) + assert job['config']['compute_type']=='cpu' + assert job['config']['gpu_count']==1 + assert response.json()['compute_type']=='cpu' + # Preview applies the same choice before compiling the spec. + from app.spec import build_spec as compile_spec + def capture(config,directory): + assert config['compute_type']=='cpu' + return compile_spec(config,directory) + monkeypatch.setattr('app.api.build_spec',capture) + assert client.post('/spec-preview',json=config,headers={'Authorization':'Bearer '+USER}).status_code==200 + + +def test_mixed_panel_keeps_gpu(service,monkeypatch): + _,db,client=service + monkeypatch.setattr('app.api.resolve_models',lambda *args:{'a':{'provider':'hf_local'},'b':'org/b'}) + response=submit(client) + assert response.status_code==202 + assert db.get(response.json()['id'])['config']['compute_type']=='gpu' + + +def test_auto_cpu_respects_cpu_limit(service): + from app.governance import PolicyUpdate + _,db,client=service + policy=db.policy();policy['policy']['limits']['max_cpu_count']=2 + db.set_policy(USER,PolicyUpdate(**policy)) + response=submit(client,payload(compute_type='gpu')) + assert response.status_code==403 + assert 'max_cpu_count' in response.json()['detail'] diff --git a/next-src/src/components/EvaluationRunner.tsx b/next-src/src/components/EvaluationRunner.tsx index 8db1a346..961e03be 100644 --- a/next-src/src/components/EvaluationRunner.tsx +++ b/next-src/src/components/EvaluationRunner.tsx @@ -37,7 +37,9 @@ export function EvaluationRunner() { const [source, setSource] = useState('airiskdilemmas'); const [count, setCount] = useState(200); const [scenarioText, setScenarioText] = useState(''); const [fileName, setFileName] = useState(''); const [ownKeys, setOwnKeys] = useState(false); const [orKey, setOrKey] = useState(''); const [rpKey, setRpKey] = useState(''); const [hfToken, setHfToken] = useState(''); - const [computeType, setComputeType] = useState('gpu'); const [gpuCount, setGPUCount] = useState(1); + const hasLocalModels = selected.some(id => models.some(m => m.id === id && m.provider === 'huggingface') || custom.some(m => m.id === id && m.provider === 'huggingface')); + const computeType = hasLocalModels ? 'gpu' : 'cpu'; + const [gpuCount, setGPUCount] = useState(1); const [cpuCount, setCPUCount] = useState(4); const [cpuFlavor, setCPUFlavor] = useState('cpu3g'); const [volume, setVolume] = useState(0); const [gpu, setGPU] = useState(gpuTypes[0]); const [disk, setDisk] = useState(100); const [stock, setStock] = useState<{ gpus: { id: string; stock: string; price_per_hour: number | null }[]; checked_at: number; disk_gb: number; gpu_count: number; min_cuda_version: string } | null>(null); @@ -144,7 +146,7 @@ export function EvaluationRunner() { setCriteria(config.criteria.join('\n')); setSource(config.scenario_source); setCount(config.scenario_count); setScenarioText(config.scenarios.map((text: string) => JSON.stringify({scenario: text})).join('\n')); setFileName('Restored scenarios'); setVisibility(config.visibility); - setOwnKeys(config.funding === 'own_keys'); setGPU(config.gpu_type); setDisk(config.disk_gb); setComputeType(config.compute_type || 'gpu'); setGPUCount(config.gpu_count || 1); setCPUCount(config.cpu_count || 4); setCPUFlavor(config.cpu_flavor || 'cpu3g'); setVolume(config.volume_gb || 0); + setOwnKeys(config.funding === 'own_keys'); setGPU(config.gpu_type); setDisk(config.disk_gb); setGPUCount(config.gpu_count || 1); setCPUCount(config.cpu_count || 4); setCPUFlavor(config.cpu_flavor || 'cpu3g'); setVolume(config.volume_gb || 0); setOrKey(''); setRpKey(''); setHfToken(''); submission.current = null; setTab('new'); setError(''); setNotice('Settings restored. Review the GPU and re-enter any personal keys, then submit a new evaluation.'); @@ -154,7 +156,7 @@ export function EvaluationRunner() { const effectiveDisk = ownKeys || isAdmin ? disk + volume : 100; const effectiveGPUCount = ownKeys || isAdmin ? gpuCount : 1; const effectiveGPU = ownKeys || isAdmin ? gpu : gpuTypes[0]; - const usesGPU = !(ownKeys || isAdmin) || computeType === 'gpu'; + const usesGPU = computeType === 'gpu'; const checkStock = useCallback(async () => { const sequence = ++stockRequest.current; setStockBusy(true); setStockError(''); setStock(null); @@ -211,11 +213,11 @@ export function EvaluationRunner() { scenario_source: source, scenario_count: count, scenarios: source === 'custom' ? parseScenarios(scenarioText,limits.max_scenarios ?? Infinity) : [], visibility, hf_token: hfToken, funding: ownKeys ? 'own_keys' : 'service', ...(ownKeys && includeKeys ? { openrouter_key: orKey, runpod_key: rpKey } : {}), - compute_type: ownKeys || isAdmin ? computeType : 'gpu', gpu_count: ownKeys || isAdmin ? (computeType === 'cpu' ? 1 : gpuCount) : 1, cpu_count: ownKeys || isAdmin ? cpuCount : 4, cpu_flavor: ownKeys || isAdmin ? cpuFlavor : 'cpu3g', volume_gb: ownKeys || isAdmin ? volume : 0, + compute_type: computeType, gpu_count: ownKeys || isAdmin ? (computeType === 'cpu' ? 1 : gpuCount) : 1, cpu_count: ownKeys || isAdmin ? cpuCount : 4, cpu_flavor: ownKeys || isAdmin ? cpuFlavor : 'cpu3g', volume_gb: ownKeys || isAdmin ? volume : 0, gpu_type: ownKeys || isAdmin ? gpu : gpuTypes[0], disk_gb: ownKeys || isAdmin ? disk : 100 }; } const blockers: string[] = []; - if ((ownKeys || isAdmin) && computeType === 'cpu' && selected.some(id => models.some(m => m.id === id && m.provider === 'huggingface') || custom.some(m => m.id === id && m.provider === 'huggingface'))) blockers.push('CPU runs support API models only. Remove Hugging Face models or select GPU.'); + if ((ownKeys || isAdmin) && computeType === 'gpu' && gpuCount > 1 && engine === 'inspect') blockers.push('Select Native to use multiple GPUs.'); if ((ownKeys || isAdmin) && limits.max_disk_gb !== null && disk + volume > limits.max_disk_gb) blockers.push('Container and workspace storage exceed your total storage limit.'); if (!name.trim()) blockers.push('Enter an evaluation name.'); @@ -319,17 +321,17 @@ export function EvaluationRunner() {
04

Compute & run

- +
- {ownKeys && <>
Keys are encrypted and removed after confirmed GPU cleanup.} - + {ownKeys && <>
Keys are encrypted and removed after confirmed compute cleanup.} +
{(['response','reflection','direct_rating'] as const).map(phase=>)}
Increase reflection tokens for reasoning models. Per-model overrides are available in Advanced configuration. {engine==='native'&&}
Compute - +
{usesGPU ? 'GPU' : 'CPU'} · automatic

{usesGPU ? 'Your panel includes a Hugging Face model, so GPU compute is required.' : 'API-only panels use CPU compute. No GPU is requested.'}

{computeType === 'gpu' ? : <> @@ -338,7 +340,7 @@ export function EvaluationRunner() { {computeType === 'gpu' && <>
GPU

{capacityMessage}

-
GPU
{gpuTypes.map(g => { const item = stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) ? stock.gpus.find(x => x.id === g) : undefined; return ; })}
GPUs are allocated together on one instance. Model size must fit the selected GPU memory.{!ownKeys && !isAdmin && ' Lab-funded runs use the default GPU.'}{stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) && ` Secure Cloud · CUDA ${stock.min_cuda_version}+ · ${stock.disk_gb} GB disk · checked ${new Date(stock.checked_at * 1000).toLocaleTimeString()}. Stock can change before allocation.`}
+
GPU
{gpuTypes.map(g => { const item = stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) ? stock.gpus.find(x => x.id === g) : undefined; return ; })}
GPUs are allocated together on one instance. Model size must fit the selected GPU memory.{!ownKeys && !isAdmin && ' LAISR Lab compute uses the default GPU.'}{stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) && ` Secure Cloud · CUDA ${stock.min_cuda_version}+ · ${stock.disk_gb} GB disk · checked ${new Date(stock.checked_at * 1000).toLocaleTimeString()}. Stock can change before allocation.`}
}
{visibility === 'public' && Completed rankings, scenarios, responses, and judgments will be visible to everyone. Public results are copied to Hugging Face. Unlisting removes them from the current listing, but repository history and downloaded copies remain public.} @@ -352,8 +354,8 @@ export function EvaluationRunner() {
Constitution
{constitution === 'custom' ? 'Custom' : label(constitution)} · {criteria.split('\n').filter(s => s.trim()).length} criteria
Scenarios
{typeof advancedCount === 'number' ? advancedCount : source === 'airiskdilemmas' ? count : 'Custom'}
Engine
{engine === 'inspect' ? 'Inspect' : 'Native EigenBench'}
-
Compute
{ownKeys ? 'Your keys' : 'Lab-funded'}
-
Compute
{(ownKeys || isAdmin) && computeType === 'cpu' ? `${cpuCount} vCPUs · ${cpuCount * (cpuFlavor === 'cpu3g' ? 4 : 2)} GB RAM` : `${ownKeys || isAdmin ? gpuCount : 1} × ${(ownKeys || isAdmin ? gpu : gpuTypes[0]).replace('NVIDIA ', '')}`}
+
Compute
{ownKeys ? 'Your keys' : 'LAISR Lab compute'}
+
Compute
{computeType === 'cpu' ? `${ownKeys || isAdmin ? cpuCount : 4} vCPUs · ${(ownKeys || isAdmin ? cpuCount : 4) * (ownKeys || isAdmin ? (cpuFlavor === 'cpu3g' ? 4 : 2) : 4)} GB RAM` : `${ownKeys || isAdmin ? gpuCount : 1} × ${(ownKeys || isAdmin ? gpu : gpuTypes[0]).replace('NVIDIA ', '')}`}
Storage
{ownKeys || isAdmin ? disk + volume : 100} GB total
Results
{visibility === 'public' ? 'Public' : 'Private'}
{!ownKeys && limits.require_credits &&
Credits
{Math.floor((credits ?? 0) / 60)} min
}