github-explore/.github/workflows/refresh-data.yml at main · AustinXT/github-explore · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
name: refresh-data

# 每日刷新选题数据源（starred 大牛快照 + trending 榜单），commit 回 main。
# 抓取 → 重建 DB 导出产物（与 pages.yml 同序）→ 提交；push src/data/** 会连带触发 pages.yml 重建站点。
on:
  schedule:
    # UTC 20:00 = 北京 04:00，错开 auto-analyze 的 UTC 0/2/4/6/8/10
    - cron: '0 20 * * *'
  workflow_dispatch:

permissions:
  contents: write

concurrency:
  group: refresh-data
  cancel-in-progress: false

jobs:
  refresh:
    runs-on: ubuntu-latest
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'
          cache-dependency-path: requirements-ci.txt

      - name: Install Python deps
        run: pip install -r requirements-ci.txt

      # ── 1. starred：遍历 users.yaml，gh api 抓每人最近 ~200 个 star → 重写 starred_seed.json
      - name: Fetch starred snapshots
        run: python3 src/scripts/fetch_starred.py

      # ── 1b. repo enrich：对所有报告 url 用 gh api 补 stars/forks/language/license/age
      #        → repo_enrich.json，供 build_reports_index.py 兜底 md 解析不到的客观字段
      - name: Fetch repo enrich
        run: python3 src/scripts/fetch_repo_enrich.py

      # ── 2. trending：浅克隆第三方 archive（所有日期 md 都在最新 commit 工作树里）
      - name: Clone trending archive
        run: |
          rm -rf /tmp/github-trending-archive
          git clone --depth 1 \
            https://github.com/duzhuoshanwai/github-trending-archive \
            /tmp/github-trending-archive

      # ── 3. 解析 archive → 重写 trending_snapshots.jsonl（END_DATE 默认到今天）
      - name: Parse trending snapshots
        run: python3 src/trending_repo/parse_trending.py

      # ── 4. 重建 DB 导出产物（与 pages.yml / CLAUDE.md 同序），
      #       让 select_next_repo.py 直接读的 all_repos_deduped.json / starred.json 同步更新
      - name: Rebuild DB exports
        run: |
          python3 src/scripts/init_db.py init
          python3 src/scripts/init_db.py seed-publish
          python3 src/scripts/build_reports_index.py
          python3 src/scripts/extract_tags.py
          python3 src/scripts/parse_starred.py
          python3 src/scripts/seed_trending.py
          python3 src/scripts/init_db.py export-json

      # ── 5. commit & push（沿用 auto-analyze.yml 的 3 次 rebase 自愈重试）
      - name: Commit & Push
        # 不开 -e：push 重试失败时本地 commit 已存，下次 cron 会带上，不应让 job fail
        run: |
          set -uo pipefail
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add src/data/ src/trending_repo/all_repos_deduped.json
          if git diff --cached --quiet; then
            echo "::notice::数据无变化，无需提交"
            exit 0
          fi
          msg=$(cat <<MSG
          chore(data): 每日刷新 starred + trending 源数据

          Run-URL ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          MSG
          )
          git commit -m "$msg"

          pushed=0
          for i in 1 2 3; do
            if git push; then
              pushed=1
              break
            fi
            echo "::warning::push 第 $i/3 次失败，尝试 rebase 后重试"
            git pull --rebase --autostash origin main \
              || echo "::warning::rebase 也失败（第 $i 次），继续重试"
            sleep $((i * 5))
          done
          if [[ $pushed -eq 0 ]]; then
            echo "::error::3 次 push 均失败，本地 commit 已保留。下次 cron 触发会自动带上。"
          fi