feat(lwe): add structured recovery challenge #107
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validate Problems | |
| on: | |
| pull_request: | |
| paths: | |
| - '.github/workflows/validate-problems.yml' | |
| - 'algorithmic/problems/**' | |
| - 'research/problems/**' | |
| - '2.0/problems/**' | |
| - 'adapters/frontier-cs-2.0/**' | |
| - 'scripts/validate_problems.py' | |
| - 'src/frontier_cs/batch/evaluator.py' | |
| - 'src/frontier_cs/config.py' | |
| - 'src/frontier_cs/runner/research_docker.py' | |
| - 'src/frontier_cs/runner/research_skypilot.py' | |
| - 'tests/test_json_language.py' | |
| - 'tests/test_frontier_cs_2_0_public_assets.py' | |
| - 'tests/test_lwe_structured_recovery_registration.py' | |
| jobs: | |
| detect-changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| algorithmic: ${{ steps.detect.outputs.algorithmic }} | |
| research: ${{ steps.detect.outputs.research }} | |
| benchmark20: ${{ steps.detect.outputs.benchmark20 }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Detect changed problems | |
| id: detect | |
| run: | | |
| # Detect changed algorithmic problems | |
| ALG=$(python scripts/detect_changed_problems.py --track algorithmic --base-ref origin/${{ github.base_ref }} -v 2>&1 | tail -1) | |
| echo "algorithmic=$ALG" >> $GITHUB_OUTPUT | |
| echo "Algorithmic problems: $ALG" | |
| # Detect changed research problems | |
| RES=$(python scripts/detect_changed_problems.py --track research --base-ref origin/${{ github.base_ref }} -v 2>&1 | tail -1) | |
| echo "research=$RES" >> $GITHUB_OUTPUT | |
| echo "Research problems: $RES" | |
| # Detect changed Frontier-CS 2.0 problems | |
| B20=$(python scripts/detect_changed_problems.py --track 2.0 --base-ref origin/${{ github.base_ref }} -v 2>&1 | tail -1) | |
| echo "benchmark20=$B20" >> $GITHUB_OUTPUT | |
| echo "Frontier-CS 2.0 problems: $B20" | |
| validate-benchmark20-infrastructure: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Run Frontier-CS 2.0 infrastructure tests | |
| run: | | |
| uv run pytest \ | |
| tests/test_json_language.py \ | |
| tests/test_frontier_cs_2_0_public_assets.py \ | |
| tests/test_lwe_structured_recovery_registration.py \ | |
| 2.0/problems/lwe_structured_recovery/tests/test_production_catalog.py \ | |
| -q | |
| - name: Check generated Frontier-CS 2.0 task compatibility | |
| run: | | |
| ERDOS_OUTPUT="$RUNNER_TEMP/frontier-cs-phase1-erdos" | |
| BBOPLACE_OUTPUT="$RUNNER_TEMP/frontier-cs-phase1-bboplace" | |
| PYTHONPATH=adapters/frontier-cs-2.0/src \ | |
| uv run --no-sync python -m frontier_cs_2_0.main \ | |
| --source "$GITHUB_WORKSPACE" \ | |
| --output-dir "$ERDOS_OUTPUT" \ | |
| --task-ids erdos_demo \ | |
| --overwrite | |
| ERDOS_TASK="$ERDOS_OUTPUT/frontier-cs-2-0-erdos-demo" | |
| test ! -d "$ERDOS_TASK/environment/harbor_app/public" | |
| if grep -Fq 'COPY harbor_app/public/ /judge/public/' \ | |
| "$ERDOS_TASK/environment/Dockerfile.judge"; then | |
| echo "Unexpected public-assets COPY in erdos_demo judge image" >&2 | |
| exit 1 | |
| fi | |
| if grep -Fq 'FRONTIER_PUBLIC_DIR' \ | |
| "$ERDOS_TASK/environment/Dockerfile.judge"; then | |
| echo "Unexpected public-assets environment in erdos_demo judge image" >&2 | |
| exit 1 | |
| fi | |
| if grep -Fq '{judge_public_assets}' \ | |
| "$ERDOS_TASK/environment/Dockerfile.judge"; then | |
| echo "Unresolved judge-public-assets placeholder in erdos_demo" >&2 | |
| exit 1 | |
| fi | |
| PYTHONPATH=adapters/frontier-cs-2.0/src \ | |
| uv run --no-sync python -m frontier_cs_2_0.main \ | |
| --source "$GITHUB_WORKSPACE" \ | |
| --output-dir "$BBOPLACE_OUTPUT" \ | |
| --task-ids \ | |
| bboplace_ispd2005 \ | |
| bboplace_iccad2015 \ | |
| bboplace_direct_ispd2005 \ | |
| bboplace_direct_iccad2015 \ | |
| --overwrite | |
| python3 tools/bboplace/check_generated_tasks.py "$BBOPLACE_OUTPUT" | |
| grep -F 'cp /solution/reference.py /app/solution.json' \ | |
| "$BBOPLACE_OUTPUT/frontier-cs-2-0-bboplace-direct-ispd2005/solution/solve.sh" | |
| grep -F 'cp /solution/reference.py /app/solution.json' \ | |
| "$BBOPLACE_OUTPUT/frontier-cs-2-0-bboplace-direct-iccad2015/solution/solve.sh" | |
| validate-algorithmic: | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.algorithmic != '' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Validate problems | |
| run: | | |
| echo "Validating algorithmic problems: ${{ needs.detect-changes.outputs.algorithmic }}" | |
| uv run python scripts/validate_problems.py \ | |
| --track algorithmic \ | |
| --problems ${{ needs.detect-changes.outputs.algorithmic }} | |
| validate-research: | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.research != '' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Setup AWS credentials | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| mkdir -p ~/.aws | |
| cat > ~/.aws/credentials << EOF | |
| [default] | |
| aws_access_key_id = $AWS_ACCESS_KEY_ID | |
| aws_secret_access_key = $AWS_SECRET_ACCESS_KEY | |
| EOF | |
| cat > ~/.aws/config << EOF | |
| [default] | |
| region = us-east-1 | |
| EOF | |
| echo "AWS credentials configured" | |
| - name: Setup GCP credentials | |
| env: | |
| GCP_CREDS: ${{ secrets.GCP_CREDENTIALS }} | |
| run: | | |
| if [ -n "$GCP_CREDS" ]; then | |
| echo "$GCP_CREDS" > /tmp/gcp-key.json | |
| echo "GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-key.json" >> $GITHUB_ENV | |
| gcloud auth activate-service-account --key-file=/tmp/gcp-key.json | |
| gcloud config set project ${{ secrets.GCP_PROJECT_ID }} | |
| echo "GCP credentials configured" | |
| fi | |
| - name: Generate SSH key for SkyPilot | |
| run: | | |
| mkdir -p ~/.ssh | |
| if [ ! -f ~/.ssh/sky-key ]; then | |
| ssh-keygen -t rsa -b 4096 -f ~/.ssh/sky-key -N "" -C "sky-ci" | |
| echo "Generated SSH key for SkyPilot" | |
| fi | |
| - name: Setup SkyPilot | |
| run: | | |
| uv run sky check aws gcp || echo "SkyPilot check failed, continuing..." | |
| - name: Validate problems | |
| timeout-minutes: 30 | |
| run: | | |
| echo "Validating research problems: ${{ needs.detect-changes.outputs.research }}" | |
| uv run python scripts/validate_problems.py \ | |
| --track research \ | |
| --timeout 1200 \ | |
| --problems ${{ needs.detect-changes.outputs.research }} \ | |
| --verbose | |
| validate-benchmark20: | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.benchmark20 != '' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Validate problems | |
| run: | | |
| echo "Validating Frontier-CS 2.0 problems: ${{ needs.detect-changes.outputs.benchmark20 }}" | |
| uv run python scripts/validate_problems.py \ | |
| --track 2.0 \ | |
| --problems ${{ needs.detect-changes.outputs.benchmark20 }} \ | |
| --verbose |