Add LLM benchmark analysis and Discord reporting

.github/workflows/llm-benchmark-periodic.yml

@@ -39,6 +39,16 @@ on:
         description: 'Run benchmarks without uploading results'
         required: false
         default: 'false'
+      skip_task_catalog_upload:
+        description: 'Skip uploading the benchmark task catalog'
+        required: false
+        type: boolean
+        default: false
+      post_discord:
+        description: 'Post the analysis summary to Discord'
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -48,9 +58,29 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  prepare_matrix:
+    name: Prepare benchmark matrix
+    runs-on: ubuntu-latest
+    outputs:
+      languages: ${{ steps.matrix.outputs.languages }}
+    steps:
+      - id: matrix
+        env:
+          INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }}
+        run: |
+          languages="$(jq -cn --arg value "$INPUT_LANGUAGES" '$value | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')"
+          echo "languages=$languages" >> "$GITHUB_OUTPUT"
+
   run-benchmarks:
+    name: Run benchmarks (${{ matrix.lang }})
+    needs: prepare_matrix
     runs-on: spacetimedb-new-runner-2
 
+    strategy:
+      fail-fast: false
+      matrix:
+        lang: ${{ fromJSON(needs.prepare_matrix.outputs.languages) }}
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -61,11 +91,13 @@ jobs:
       - uses: Swatinem/rust-cache@v2
 
       - name: Setup .NET SDK
+        if: matrix.lang == 'csharp'
         uses: actions/setup-dotnet@v4
         with:
           global-json-file: global.json
 
       - name: Install WASI workload
+        if: matrix.lang == 'csharp'
         env:
           DOTNET_MULTILEVEL_LOOKUP: "0"
           DOTNET_CLI_HOME: ${{ runner.temp }}/dotnet-home
@@ -74,25 +106,25 @@ jobs:
           dotnet workload install wasi-experimental --skip-manifest-update --disable-parallel
 
       - name: Pack C# runtime packages
-        if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'csharp') }}
+        if: matrix.lang == 'csharp'
         run: |
           dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime
           dotnet pack -c Release crates/bindings-csharp/Runtime
 
       - name: Set up Node.js
-        if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }}
+        if: matrix.lang == 'typescript'
         uses: actions/setup-node@v4
         with:
           node-version: 22
 
       - name: Install pnpm
-        if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }}
+        if: matrix.lang == 'typescript'
         uses: ./.github/actions/setup-pnpm
         with:
           run_install: true
 
       - name: Build TypeScript SDK
-        if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }}
+        if: matrix.lang == 'typescript'
         run: pnpm build
         working-directory: crates/bindings-typescript
 
@@ -119,22 +151,28 @@ jobs:
           DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1"
           MSBUILDDISABLENODEREUSE: "1"
           DOTNET_CLI_USE_MSBUILD_SERVER: "0"
-          LLM_BENCH_CSHARP_CONCURRENCY: "1"
-          INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }}
+          LLM_BENCH_CONCURRENCY: "8"
+          LLM_BENCH_RUST_CONCURRENCY: "4"
+          LLM_BENCH_CSHARP_CONCURRENCY: "2"
+          LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }}
+          LLM_BENCHMARK_REPORT_DIR: ${{ runner.temp }}/llm-benchmark-reports
+          INPUT_LANGUAGE: ${{ matrix.lang }}
           INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }}
           INPUT_MODELS: ${{ inputs.models || '' }}
           INPUT_MODES: ${{ inputs.modes || 'guidelines,no_context' }}
           INPUT_CATEGORIES: ${{ inputs.categories || '' }}
           INPUT_TASKS: ${{ inputs.tasks || '' }}
           INPUT_DRY_RUN: ${{ inputs.dry_run || 'false' }}
+          INPUT_SKIP_TASK_CATALOG_UPLOAD: ${{ inputs.skip_task_catalog_upload || 'false' }}
         run: |
-          LANGS="$INPUT_LANGUAGES"
+          LANG="$INPUT_LANGUAGE"
           MODEL_SET="$INPUT_MODEL_SET"
           MODELS="$INPUT_MODELS"
           MODES="$INPUT_MODES"
           CATEGORIES="$INPUT_CATEGORIES"
           TASKS="$INPUT_TASKS"
           DRY_RUN="$INPUT_DRY_RUN"
+          SKIP_TASK_CATALOG_UPLOAD="$INPUT_SKIP_TASK_CATALOG_UPLOAD"
 
           case "$MODEL_SET" in
             website_active)
@@ -162,45 +200,102 @@ jobs:
               ;;
           esac
 
-          SUCCEEDED=0
-          FAILED=0
-          for LANG in $(echo "$LANGS" | tr ',' ' '); do
-            EXTRA_ARGS=()
-            if [ -n "$CATEGORIES" ]; then
-              EXTRA_ARGS+=(--categories "$CATEGORIES")
-            fi
-            if [ -n "$TASKS" ]; then
-              EXTRA_ARGS+=(--tasks "$TASKS")
-            fi
-            if [ "$DRY_RUN" = "true" ]; then
-              EXTRA_ARGS+=(--dry-run)
-            fi
-
-            if [ "$MODEL_SET" = "website_active" ]; then
-              if llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}"; then
-                SUCCEEDED=$((SUCCEEDED + 1))
-              else
-                echo "::warning::Benchmark run failed for lang=$LANG"
-                FAILED=$((FAILED + 1))
-              fi
-            elif [ "$MODEL_SET" = "local_defaults" ]; then
-              if llm_benchmark run --lang "$LANG" --modes "$MODES" "${EXTRA_ARGS[@]}"; then
-                SUCCEEDED=$((SUCCEEDED + 1))
-              else
-                echo "::warning::Benchmark run failed for lang=$LANG"
-                FAILED=$((FAILED + 1))
-              fi
-            else
-              if llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}"; then
-                SUCCEEDED=$((SUCCEEDED + 1))
-              else
-                echo "::warning::Benchmark run failed for lang=$LANG models=$MODELS"
-                FAILED=$((FAILED + 1))
-              fi
-            fi
+          EXTRA_ARGS=()
+          if [ -n "$CATEGORIES" ]; then
+            EXTRA_ARGS+=(--categories "$CATEGORIES")
+          fi
+          if [ -n "$TASKS" ]; then
+            EXTRA_ARGS+=(--tasks "$TASKS")
+          fi
+          if [ "$DRY_RUN" = "true" ]; then
+            EXTRA_ARGS+=(--dry-run)
+          fi
+          if [ "$SKIP_TASK_CATALOG_UPLOAD" = "true" ]; then
+            EXTRA_ARGS+=(--skip-task-catalog-upload)
+          fi
+
+          if [ "$MODEL_SET" = "website_active" ]; then
+            llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}"
+          elif [ "$MODEL_SET" = "local_defaults" ]; then
+            llm_benchmark run --lang "$LANG" --modes "$MODES" "${EXTRA_ARGS[@]}"
+          else
+            llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}"
+          fi
+
+      - name: Upload analysis reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: llm-benchmark-analysis-${{ matrix.lang }}
+          path: ${{ runner.temp }}/llm-benchmark-reports
+          if-no-files-found: ignore
+          retention-days: 14
+
+  summarize:
+    name: Summarize benchmark analysis
+    if: always()
+    needs: run-benchmarks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Test Discord formatter
+        run: python3 tools/xtask-llm-benchmark/scripts/test_discord_summary.py
+
+      - name: Download analysis reports
+        continue-on-error: true
+        uses: actions/download-artifact@v4
+        with:
+          pattern: llm-benchmark-analysis-*
+          path: reports
+          merge-multiple: true
+
+      - name: Publish workflow summary
+        run: |
+          echo "# LLM benchmark analysis" >> "$GITHUB_STEP_SUMMARY"
+          echo >> "$GITHUB_STEP_SUMMARY"
+
+          if [ ! -d reports ] || ! find reports -type f -name '*.md' -print -quit | grep -q .; then
+            echo "No analysis reports were produced." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          echo "Full per-model reports are available in the workflow artifacts." >> "$GITHUB_STEP_SUMMARY"
+          echo >> "$GITHUB_STEP_SUMMARY"
+
+          find reports -type f -name '*.md' -print | sort | while IFS= read -r report; do
+            awk '/^## Failure patterns/{exit} NR <= 80 {print}' "$report" >> "$GITHUB_STEP_SUMMARY"
+            echo >> "$GITHUB_STEP_SUMMARY"
+            echo "---" >> "$GITHUB_STEP_SUMMARY"
+            echo >> "$GITHUB_STEP_SUMMARY"
           done
-          echo "Benchmark runs: $SUCCEEDED succeeded, $FAILED failed"
-          if [ "$FAILED" -gt 0 ]; then
-            echo "::error::$FAILED benchmark run(s) failed"
-            exit 1
+
+      - name: Prepare Discord summary
+        if: github.event_name == 'schedule' || inputs.post_discord
+        env:
+          RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          RUN_LABEL: ${{ github.event_name == 'schedule' && 'Weekly scheduled run' || 'Manual run' }}
+        run: >
+          python3 tools/xtask-llm-benchmark/scripts/discord_summary.py
+          --reports-dir reports
+          --run-url "$RUN_URL"
+          --run-label "$RUN_LABEL"
+          --output discord-payload.json
+
+      - name: Post Discord summary
+        if: github.event_name == 'schedule' || inputs.post_discord
+        continue-on-error: true
+        env:
+          DISCORD_WEBHOOK_URL: ${{ secrets.LLM_BENCHMARK_DISCORD_WEBHOOK_URL }}
+        run: |
+          if [ -z "$DISCORD_WEBHOOK_URL" ]; then
+            echo "::warning::LLM_BENCHMARK_DISCORD_WEBHOOK_URL is not configured"
+            exit 0
           fi
+
+          curl --fail --silent --show-error \
+            --header 'Content-Type: application/json' \
+            --data-binary @discord-payload.json \
+            "$DISCORD_WEBHOOK_URL"