From 5d6b67aa05f360f365f7fa70ed6bc084374c33cd Mon Sep 17 00:00:00 2001 From: invi-bhagyesh <104337027+invi-bhagyesh@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:47:35 +0530 Subject: [PATCH] Check GPU capacity automatically and explain allocation failures --- backend/app/progress.py | 7 +++- backend/app/runpod.py | 2 +- backend/tests/test_backend.py | 2 ++ backend/tests/test_runpod.py | 22 ++++++++++-- next-src/src/app/pages.css | 3 ++ next-src/src/components/EvaluationRunner.tsx | 38 ++++++++++++++------ 6 files changed, 58 insertions(+), 16 deletions(-) diff --git a/backend/app/progress.py b/backend/app/progress.py index 2c5a8c47..53eaa379 100644 --- a/backend/app/progress.py +++ b/backend/app/progress.py @@ -20,6 +20,8 @@ def progress(job, now=None): now = int(time.time()) if now is None else now state = job['state']; stage = job['stage'] error = job.get('error_code') or '' + config = job.get('config', {}) + gpu_label = f"{config.get('gpu_count', 1)} × {config.get('gpu_type', 'selected GPU')}" provider_error = error.startswith('provider_create_') provider_detail = ('RunPod returned HTTP '+error.removeprefix('provider_create_http_')+'. ' if error.startswith('provider_create_http_') else 'The RunPod allocation request did not return a confirmed result. ') provider_detail += 'No GPU allocation has been confirmed. The scheduler checks for a late-created instance instead of sending another request.' @@ -29,7 +31,7 @@ def progress(job, now=None): allocation = job.get('allocation') or {} if allocation.get('outcome') == 'unavailable' and not job.get('pod_id'): delay = max(0, (allocation.get('next_retry_at') or now) - now) - title, detail, step = 'Waiting for GPU availability', f"Attempt {allocation['attempts']}: selected GPU unavailable. Next retry in {delay} seconds; retrying for up to 5 minutes.", 1 + title, detail, step = 'Waiting for GPU availability', f"RunPod has no matching capacity for {gpu_label}. Attempt {allocation['attempts']}. Next retry in {delay} seconds, for up to 5 minutes. No GPU has been allocated and model evaluation has not started. You can cancel and choose another GPU.", 1 elif provider_error and not job.get('pod_id'): title, detail, step = 'GPU allocation not confirmed', provider_detail, 1 elif job.get('pod_id'): @@ -47,6 +49,9 @@ def progress(job, now=None): title, detail, step = 'Completed', 'Rankings and transcripts are ready.', 6 else: title, detail, step = ('Cancelled' if state == 'cancelled' else 'Failed'), (provider_detail if provider_error else ERRORS.get(job.get('error_code'), 'The evaluation stopped. See worker output for details.')), -1 + if state == 'failed' and error == 'gpu_unavailable': + title = 'GPU capacity unavailable' + detail = f'RunPod could not allocate {gpu_label} within 5 minutes. Model evaluation never started. Retry with another GPU or try again when capacity returns.' if job.get('config', {}).get('compute_type') == 'cpu': title = title.replace('GPU', 'CPU') detail = detail.replace('GPU', 'CPU') diff --git a/backend/app/runpod.py b/backend/app/runpod.py index 542834ee..a35be0a8 100644 --- a/backend/app/runpod.py +++ b/backend/app/runpod.py @@ -31,7 +31,7 @@ def gpu_availability(disk_gb=100, gpu_count=1): rows = [] for gpu in body['data']['gpuTypes']: price = gpu.get('lowestPrice') or {} - stock = price.get('stockStatus') + stock = price.get('stockStatus') if gpu.get('lowestPrice') is not None else 'None' rows.append({'id': gpu['id'], 'stock': stock if stock in ('High','Medium','Low','None') else 'Unknown', 'price_per_hour': price.get('uninterruptablePrice')}) return {'gpus': rows, 'checked_at': int(time.time()), 'min_cuda_version': MIN_CUDA_VERSION, diff --git a/backend/tests/test_backend.py b/backend/tests/test_backend.py index 1d5e7894..e521825b 100644 --- a/backend/tests/test_backend.py +++ b/backend/tests/test_backend.py @@ -332,6 +332,8 @@ def unavailable(job): raise GPUUnavailable() assert view['progress']['title'] == 'Waiting for GPU availability' assert 'Attempt 1' in view['progress']['detail'] assert 'Next retry' in view['progress']['detail'] + assert 'NVIDIA A40' in view['progress']['detail'] + assert 'model evaluation has not started' in view['progress']['detail'] def test_compute_limits_and_cpu_local_models(service, monkeypatch): diff --git a/backend/tests/test_runpod.py b/backend/tests/test_runpod.py index 05607b4a..3d7f8233 100644 --- a/backend/tests/test_runpod.py +++ b/backend/tests/test_runpod.py @@ -21,18 +21,19 @@ def handle(request): assert pods.create({'id':str(uuid4()),'config':{}})=='mock-pod' -def test_stock_check_preserves_unknown_and_filters_cuda_disk(monkeypatch): +def test_stock_check_reports_missing_offers_and_filters_cuda_disk(monkeypatch): from app.runpod import gpu_availability def post(url, json, timeout): assert json['variables']['input'] == {'gpuCount': 1, 'secureCloud': True, 'minCudaVersion': '13.0', 'minDisk': 200} return httpx.Response(200, request=httpx.Request('POST',url), json={'data':{'gpuTypes':[ {'id':'A40','lowestPrice':{'stockStatus':'Low','uninterruptablePrice':0.49}}, {'id':'A100','lowestPrice':None}, - {'id':'H100','lowestPrice':{'stockStatus':'None','uninterruptablePrice':None}} + {'id':'H100','lowestPrice':{'stockStatus':'None','uninterruptablePrice':None}}, + {'id':'unknown','lowestPrice':{'stockStatus':'unexpected','uninterruptablePrice':None}} ]}}) monkeypatch.setattr('app.runpod.httpx.post',post) result=gpu_availability(200) - assert [g['stock'] for g in result['gpus']] == ['Low','Unknown','None'] + assert [g['stock'] for g in result['gpus']] == ['Low','None','None','Unknown'] assert result['disk_gb']==200 @@ -86,3 +87,18 @@ def handle(request): with httpx.Client(transport=httpx.MockTransport(handle)) as client: pods.client.close(); pods.client=client assert pods.create({'id':str(uuid4()),'config':{'gpu_count':4,'volume_gb':200}})=='multi-pod' + + +def test_every_selectable_gpu_is_forwarded_without_substitution(): + from typing import get_args + from app.models import EvaluationRequest + for gpu in get_args(EvaluationRequest.model_fields['gpu_type'].annotation): + pods=RunPod(Settings(environment='test',worker_secret='x'*40)) + def handle(request): + payload=json.loads(request.content)['variables']['input'] + assert payload['gpuTypeId']==gpu + assert payload['gpuCount']==1 + return httpx.Response(200,json={'data':{'podFindAndDeployOnDemand':{'id':'test-pod'}}}) + with httpx.Client(transport=httpx.MockTransport(handle)) as client: + pods.client.close();pods.client=client + assert pods.create({'id':str(uuid4()),'config':{'gpu_type':gpu,'gpu_count':1}})=='test-pod' diff --git a/next-src/src/app/pages.css b/next-src/src/app/pages.css index fc20b7a5..e85fa72d 100644 --- a/next-src/src/app/pages.css +++ b/next-src/src/app/pages.css @@ -1005,3 +1005,6 @@ body:has(.arena-home) .page-hills { transition: none; } } + +.eval-capacity-note { padding: 12px 16px; border: 1px solid var(--border); border-radius: 12px; font-size: 13px; line-height: 1.6; background: var(--bg-elevated); } +.eval-capacity-note[data-warning='true'] { border-color: #af874b; border-left-width: 3px; } diff --git a/next-src/src/components/EvaluationRunner.tsx b/next-src/src/components/EvaluationRunner.tsx index c089e36e..8db1a346 100644 --- a/next-src/src/components/EvaluationRunner.tsx +++ b/next-src/src/components/EvaluationRunner.tsx @@ -1,6 +1,6 @@ 'use client'; -import { useEffect, useRef, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import type { Session } from '@supabase/supabase-js'; import { evaluationAPI, evaluationAuth, evaluationRequest as request, type EvaluationJob as Job } from '@/lib/evaluation'; import { CONSTITUTIONS_DATA } from '@/lib/constitutions-data'; @@ -40,7 +40,8 @@ export function EvaluationRunner() { const [computeType, setComputeType] = useState('gpu'); const [gpuCount, setGPUCount] = useState(1); const [cpuCount, setCPUCount] = useState(4); const [cpuFlavor, setCPUFlavor] = useState('cpu3g'); const [volume, setVolume] = useState(0); const [gpu, setGPU] = useState(gpuTypes[0]); const [disk, setDisk] = useState(100); - const [stock, setStock] = useState<{ gpus: { id: string; stock: string; price_per_hour: number | null }[]; checked_at: number; disk_gb: number; gpu_count: number } | null>(null); + const [stock, setStock] = useState<{ gpus: { id: string; stock: string; price_per_hour: number | null }[]; checked_at: number; disk_gb: number; gpu_count: number; min_cuda_version: string } | null>(null); + const stockRequest = useRef(0); const [stockBusy, setStockBusy] = useState(false); const [stockError, setStockError] = useState(''); const [visibility, setVisibility] = useState('private'); const [accessError, setAccessError] = useState(''); @@ -150,14 +151,29 @@ export function EvaluationRunner() { } catch (e) { setError((e as Error).message); } } - async function checkStock() { + const effectiveDisk = ownKeys || isAdmin ? disk + volume : 100; + const effectiveGPUCount = ownKeys || isAdmin ? gpuCount : 1; + const effectiveGPU = ownKeys || isAdmin ? gpu : gpuTypes[0]; + const usesGPU = !(ownKeys || isAdmin) || computeType === 'gpu'; + const checkStock = useCallback(async () => { + const sequence = ++stockRequest.current; setStockBusy(true); setStockError(''); setStock(null); try { - const response = await request(`/compute/availability?disk_gb=${ownKeys || isAdmin ? disk + volume : 100}&gpu_count=${ownKeys || isAdmin ? gpuCount : 1}`); - setStock(await response.json()); - } catch (e) { setStockError((e as Error).message); } - finally { setStockBusy(false); } - } + const response = await request(`/compute/availability?disk_gb=${effectiveDisk}&gpu_count=${effectiveGPUCount}`); + const data = await response.json(); + if (sequence === stockRequest.current) setStock(data); + } catch (e) { if (sequence === stockRequest.current) setStockError((e as Error).message); } + finally { if (sequence === stockRequest.current) setStockBusy(false); } + }, [effectiveDisk, effectiveGPUCount]); + useEffect(() => { + if (access !== 'approved' || !usesGPU || tab !== 'new') return; + const debounce = setTimeout(() => void checkStock(), 400); + const timer = setInterval(() => void checkStock(), 60000); + return () => { clearTimeout(debounce); clearInterval(timer); ++stockRequest.current; setStockBusy(false); }; + }, [access, usesGPU, tab, session?.user.id, checkStock]); + const matchingStock = stock && stock.disk_gb === effectiveDisk && stock.gpu_count === effectiveGPUCount ? stock : null; + const selectedStock = matchingStock?.gpus.find(item => item.id === effectiveGPU); + const capacityMessage = stockBusy ? 'Checking matching RunPod capacity…' : stockError ? 'Availability could not be checked. Allocation may still fail; refresh or submit to try.' : selectedStock?.stock === 'None' ? 'No matching capacity reported. Choose another GPU, or submit to retry allocation for up to 5 minutes. Evaluation starts only after a GPU is allocated.' : selectedStock?.stock === 'Low' ? 'Limited capacity. Allocation can fail even when stock is reported; we retry for up to 5 minutes.' : !selectedStock || selectedStock.stock === 'Unknown' ? 'Capacity is unconfirmed. RunPod has not reported matching stock; submitting will attempt allocation.' : 'Capacity reported available. This is not a reservation; RunPod confirms availability when allocating your instance.'; const collectionOptions = (advancedOptions.collection || {}) as {generation?:Record;failure_policy?:string}; function setBudget(phase:string,value:string) { @@ -320,9 +336,9 @@ export function EvaluationRunner() { } {computeType === 'gpu' && <> -
GPU
- {stockError &&

{stockError}

} -
GPU
{gpuTypes.map(g => { const item = stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) ? stock.gpus.find(x => x.id === g) : undefined; return ; })}
GPUs are allocated together on one instance.{!ownKeys && !isAdmin && ' Lab-funded runs use the default GPU.'}{stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) && ` Secure Cloud · CUDA 13+ · ${stock.disk_gb} GB disk · checked ${new Date(stock.checked_at * 1000).toLocaleTimeString()}. Stock can change before allocation.`}
+
GPU
+

{capacityMessage}

+
GPU
{gpuTypes.map(g => { const item = stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) ? stock.gpus.find(x => x.id === g) : undefined; return ; })}
GPUs are allocated together on one instance. Model size must fit the selected GPU memory.{!ownKeys && !isAdmin && ' Lab-funded runs use the default GPU.'}{stock && stock.disk_gb === (ownKeys || isAdmin ? disk + volume : 100) && stock.gpu_count === (ownKeys || isAdmin ? gpuCount : 1) && ` Secure Cloud · CUDA ${stock.min_cuda_version}+ · ${stock.disk_gb} GB disk · checked ${new Date(stock.checked_at * 1000).toLocaleTimeString()}. Stock can change before allocation.`}
}
{visibility === 'public' && Completed rankings, scenarios, responses, and judgments will be visible to everyone. Public results are copied to Hugging Face. Unlisting removes them from the current listing, but repository history and downloaded copies remain public.}