From 9c63df43fceef2c314f08c0c0105d5d4f7ea7476 Mon Sep 17 00:00:00 2001
From: wms2537 <51080539+wms2537@users.noreply.github.com>
Date: Wed, 19 Aug 2026 10:56:40 +0800
Subject: [PATCH 1/2] research: add JumpBench model-discovery workflow

---
 .github/workflows/jumpbench-neural-eval.yml | 54 +++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 .github/workflows/jumpbench-neural-eval.yml

diff --git a/.github/workflows/jumpbench-neural-eval.yml b/.github/workflows/jumpbench-neural-eval.yml
new file mode 100644
index 0000000..063b010
--- /dev/null
+++ b/.github/workflows/jumpbench-neural-eval.yml
@@ -0,0 +1,54 @@
+name: JumpBench neural evaluation
+
+on:
+  push:
+    branches:
+      - research/jumpbench-neural-eval
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  discover-models:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Environment
+        run: |
+          python --version
+          free -h
+          nproc
+      - name: Discover LittleLearner model repositories
+        run: |
+          set -euo pipefail
+          python - <<'PY'
+          import json, urllib.parse, urllib.request
+          queries = [
+              {"author": "littlelearner", "limit": 100, "full": "true"},
+              {"search": "LittleLearner", "limit": 100, "full": "true"},
+              {"search": "littlelearner-ll", "limit": 100, "full": "true"},
+              {"filter": "arxiv:2608.13545", "limit": 100, "full": "true"},
+          ]
+          found = {}
+          for params in queries:
+              url = "https://huggingface.co/api/models?" + urllib.parse.urlencode(params)
+              print("QUERY", url)
+              with urllib.request.urlopen(url, timeout=60) as response:
+                  payload = json.load(response)
+              print("COUNT", len(payload))
+              for model in payload:
+                  model_id = model.get("id") or model.get("modelId")
+                  if model_id:
+                      found[model_id] = model
+                      print("MODEL", model_id, "sha=", model.get("sha"), "private=", model.get("private"))
+          with open("hf_models.json", "w", encoding="utf-8") as handle:
+              json.dump(found, handle, indent=2)
+          print("UNIQUE_MODELS", len(found))
+          PY
+      - uses: actions/upload-artifact@v4
+        with:
+          name: jumpbench-hf-model-discovery
+          path: hf_models.json
+          if-no-files-found: error

From 3d810bf5979a7a8c4b71b5b0379dde5461076474 Mon Sep 17 00:00:00 2001
From: wms2537 <51080539+wms2537@users.noreply.github.com>
Date: Wed, 19 Aug 2026 10:57:09 +0800
Subject: [PATCH 2/2] research: trigger JumpBench evaluation on pull requests

---
 .github/workflows/jumpbench-neural-eval.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/jumpbench-neural-eval.yml b/.github/workflows/jumpbench-neural-eval.yml
index 063b010..ca329b3 100644
--- a/.github/workflows/jumpbench-neural-eval.yml
+++ b/.github/workflows/jumpbench-neural-eval.yml
@@ -4,6 +4,9 @@ on:
   push:
     branches:
       - research/jumpbench-neural-eval
+  pull_request:
+    branches:
+      - master
   workflow_dispatch:
 
 permissions: