ci: report Experimental Tests and Python Compatibility to Discussions, with step summaries (#2030)

.github/workflows/ci-reports.yml

@@ -0,0 +1,97 @@
+name: CI Reports
+
+# #2030: posts the runs of Experimental Tests and Python Compatibility that have
+# no pull request to comment on, a push or the schedule, to their threads in the
+# Discussions category REPORT_CATEGORY names, as coverage-comment.yml's
+# `discussion` job posts Unit Tests'. A `workflow_run` relay, so neither
+# workflow's own jobs gain a write scope, and every report is read when its run
+# has finished.
+#
+# `workflow_run` runs this file as it is on the default branch. The job checks
+# out the default branch's scripts alone, never the run's head; the run's
+# fields reach the scripts as `env:`, and its jobs' summaries, from their
+# `job-summary-*` artifacts, are data the report quotes.
+on:
+  workflow_run:
+    workflows: ["Experimental Tests", "Python Compatibility"]
+    types: [completed]
+
+# The job holds its own scopes.
+permissions: {}
+
+# Each relayed run's group is its own: a group cancels a pending run when a
+# newer one joins it, which would drop that run's report.
+concurrency:
+  group: ci-reports-${{ github.event.workflow_run.id }}
+  cancel-in-progress: false
+
+# One Discussions category holds every report with no pull request to comment
+# on, one thread per job.
+env:
+  REPORT_CATEGORY: CI reports
+
+jobs:
+  # A push run posts as `push` to `<workflow file stem>: <branch> push runs`,
+  # and a scheduled one as `scheduled` to `... scheduled runs`;
+  # util/check_suite_report.py names the thread and the mode. Never a pull
+  # request's run, whose checks are its report. The steps are
+  # coverage-comment.yml `discussion`'s, which
+  # tests/project/test_unit_tests_push_reports_2030.py holds identical. The job
+  # and its steps are `continue-on-error`; both scripts warn instead of failing.
+  discussion:
+    name: Post the run report
+    if: ${{ github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    continue-on-error: true
+    permissions:
+      actions: read
+      checks: read
+      contents: read
+      discussions: write
+
+    steps:
+      # The default branch, which is what `workflow_run` checks out, and only
+      # the scripts; nothing from the run's head.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          sparse-checkout: |
+            util/_actions.py
+            util/check_suite_report.py
+            util/discussion_report.py
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+
+      # The run's fields are data, in `env:`, where the script reads them.
+      - name: Write the run report
+        id: report
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          WORKFLOW: ${{ github.event.workflow_run.name }}
+          SUITE: ${{ github.event.workflow_run.check_suite_node_id }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+          RUN_URL: ${{ github.event.workflow_run.html_url }}
+          RUN_EVENT: ${{ github.event.workflow_run.event }}
+          RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+          HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
+        run: python3 util/check_suite_report.py --report report/run-report.md
+
+      # The script labels the run it links with GITHUB_WORKFLOW, which names
+      # this workflow, so the command sets the relayed run's for that process.
+      - name: Post the report to Discussions
+        if: ${{ steps.report.outputs.post == 'true' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          WORKFLOW: ${{ github.event.workflow_run.name }}
+          TITLE: ${{ steps.report.outputs.title }}
+          MODE: ${{ steps.report.outputs.mode }}
+          STATUS: ${{ steps.report.outputs.status }}
+          RUN_URL: ${{ github.event.workflow_run.html_url }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          GITHUB_WORKFLOW="$WORKFLOW" python3 util/discussion_report.py --category "$REPORT_CATEGORY" \
+            --title "$TITLE" --report report/run-report.md \
+            --mode "$MODE" --status "$STATUS" --run-url "$RUN_URL" --sha "$HEAD_SHA"

.github/workflows/coverage-comment.yml

@@ -18,9 +18,12 @@ name: Coverage Comment
 #
 # A push run has no pull request to comment on, so the `discussion` job posts it
 # to its branch's thread in the Discussions category REPORT_CATEGORY names
-# (#2030). It reads the run's jobs through the API, not the run's artifacts,
-# runs util/discussion_report.py from the default branch's checkout, never the
-# run's head, and alone holds `discussions: write`.
+# (#2030). It reads the run's jobs through the API and the jobs' summaries from
+# their `job-summary-*` artifacts, as data the report quotes, runs
+# util/check_suite_report.py and util/discussion_report.py from the default
+# branch's checkout, never the run's head, and alone holds `discussions: write`.
+# ci-reports.yml's `discussion` job runs the same steps for Experimental Tests
+# and Python Compatibility.
 on:
   workflow_run:
     workflows: ["Unit Tests"]
@@ -133,156 +136,66 @@ jobs:
 
   # #2030: a Unit Tests run on a push, posted as a `push` report to the thread
   # `unit-tests: <branch> push runs`: a run that failed or warned posts, then one
-  # green run after it. The report is built from the run's check suite, read in
-  # one GraphQL call rather than from an artifact, and quotes each job name and
-  # annotation as a code span, so none renders or mentions. A green run reports
-  # `warning` where a job failed without failing it, or a job that passed holds
-  # a warning or failure annotation, as a failing KTLO leg does (#1672), other
-  # than those `EXPECTED` lists as written on every run by design. The job and
-  # its steps are `continue-on-error`; util/discussion_report.py warns instead.
+  # green run after it. util/check_suite_report.py writes the report from the
+  # run's check suite and its jobs' summaries; ci-reports.yml's `discussion` job
+  # runs the same steps, which tests/project/test_unit_tests_push_reports_2030.py
+  # holds identical. The job and its steps are `continue-on-error`; both scripts
+  # warn instead of failing.
   discussion:
     name: Post the push run report
     if: ${{ github.event.workflow_run.event == 'push' }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     continue-on-error: true
     permissions:
+      actions: read
       checks: read
       contents: read
       discussions: write
 
     steps:
       # The default branch, which is what `workflow_run` checks out, and only
-      # the script; nothing from the run's head.
+      # the scripts; nothing from the run's head.
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           sparse-checkout: |
+            util/_actions.py
+            util/check_suite_report.py
             util/discussion_report.py
           sparse-checkout-cone-mode: false
           persist-credentials: false
 
+      # The run's fields are data, in `env:`, where the script reads them.
       - name: Write the run report
         id: report
         continue-on-error: true
         env:
           GH_TOKEN: ${{ github.token }}
+          WORKFLOW: ${{ github.event.workflow_run.name }}
           SUITE: ${{ github.event.workflow_run.check_suite_node_id }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
           RUN_URL: ${{ github.event.workflow_run.html_url }}
+          RUN_EVENT: ${{ github.event.workflow_run.event }}
           RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
           HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
           HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
-        run: |
-          set -u
-          # The run's conclusion as a --status; anything else, `timed_out` and
-          # `startup_failure` among them, is a failure.
-          case "$RUN_CONCLUSION" in
-            success) status=success ;;
-            cancelled|stale) status=cancelled ;;
-            skipped|neutral) status=skipped ;;
-            *) status=failure ;;
-          esac
-          branch=$(printf '%s' "$HEAD_BRANCH" | tr -c 'A-Za-z0-9._/-' '-')
-          echo "title=unit-tests: ${branch:-unknown} push runs" >> "$GITHUB_OUTPUT"
-
-          # shellcheck disable=SC2016 # `$id` is a GraphQL variable
-          query='query($id: ID!) { node(id: $id) { ... on CheckSuite { checkRuns(first: 100) {
-            pageInfo { hasNextPage }
-            nodes { name conclusion annotations(first: 20) { nodes { annotationLevel title message } } } } } } }'
-          if ! gh api graphql -f query="$query" -f id="$SUITE" > "$RUNNER_TEMP/jobs.json"; then
-            echo "::warning::The run's jobs could not be read, so the report names none of them."
-            : > "$RUNNER_TEMP/jobs.json"
-          fi
-
-          mkdir -p report
-          if ! STATUS="$status" python3 - "$RUNNER_TEMP/jobs.json" report/unit-tests-report.md <<'PY'
-          import json, os, re, sys
-          status, conclusion = os.environ['STATUS'], os.environ['RUN_CONCLUSION']
-          sha, url = os.environ['HEAD_SHA'], os.environ['RUN_URL']
-
-          def code(text):
-              """One line of ``text``, at most 200 characters, as a code span, so it renders and mentions nothing."""
-              text = ' '.join(str(text).replace('`', "'").split())
-              return '`{}`'.format(text if len(text) <= 200 else text[:197] + '...')
-
-          # The annotations a job writes on every run by design, which neither
-          # count nor are listed: per job, `(level, title prefix)`, where an
-          # empty prefix matches only an empty title, the runner's own line for
-          # the step that failed. Matched on the title, not the message, which
-          # #2041 rewords. Anything else in these jobs, a failed tests step
-          # among it, still makes a green run `warning`.
-          # None today: the 3.7 legs that stopped at install went with #2051, and
-          # pypy3.10's installs since #2040's cryptography cap.
-          EXPECTED = {}
-
-          def expected(job, note):
-              """Whether ``note``, an annotation of ``job``, is one :data:`EXPECTED` lists."""
-              title = note.get('title') or ''
-              return any(note.get('annotationLevel') == level and (title.startswith(prefix) if prefix else not title)
-                         for level, prefix in EXPECTED.get(job, ()))
-
-          try:
-              with open(sys.argv[1], encoding='utf-8') as file:
-                  runs = json.load(file)['data']['node']['checkRuns']
-              nodes, more = [node for node in runs['nodes'] if node], runs['pageInfo']['hasNextPage']
-          except (OSError, ValueError, LookupError, TypeError):
-              nodes, more = None, False
-          failed, warned, by_design = [], [], False
-          for node in nodes or []:
-              name, result = node.get('name') or '?', (node.get('conclusion') or 'unknown').lower()
-              if result not in ('success', 'skipped', 'neutral'):
-                  failed.append(f'{code(name)} ({result})')
-              elif result == 'success':
-                  flagged = [note for note in (node.get('annotations') or {}).get('nodes') or []
-                             if note and note.get('annotationLevel') in ('WARNING', 'FAILURE')]
-                  kept = [note for note in flagged if not expected(name, note)]
-                  by_design = by_design or len(kept) < len(flagged)
-                  notes = [note.get('title') or note.get('message') or '' for note in kept]
-                  notes = list(dict.fromkeys(note for note in notes if note.strip()))[:5]
-                  if notes:
-                      warned.append(f'{code(name)}: ' + '; '.join(map(code, notes)))
-          if status == 'success' and (failed or warned):
-              status = 'warning'
-
-          run = 'The Unit Tests run'
-          if re.fullmatch(r'https://[\w.-]+/[\w.-]+/[\w.-]+/actions/runs/\d+', url):
-              run = f'[Unit Tests run]({url})'
-          commit = f' for `{sha[:9]}`' if re.fullmatch(r'[0-9a-f]{40}', sha) else ''
-          lines = ['### Unit Tests push run', '', f'{run}{commit} concluded {code(conclusion or "unknown")}.']
-          if failed:
-              heading = 'Failed without failing the run' if conclusion == 'success' else 'Did not pass'
-              lines += ['', f'**{heading}:**', '', *(f'- {item}' for item in failed)]
-          if warned:
-              lines += ['', '**Warned, in jobs that passed:**', '', *(f'- {item}' for item in warned)]
-          if nodes is None:
-              lines += ['', "The run's jobs could not be read, so none is named here."]
-          elif more:
-              lines += ['', 'Only the first 100 jobs were read.']
-          elif not failed and not warned:
-              lines += ['', 'Every job passed, with no warning but the by-design ones.' if by_design
-                        else 'Every job passed, with no warning.']
-          with open(sys.argv[2], 'w', encoding='utf-8') as file:
-              print('\n'.join(lines), file=file)
-          with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as file:
-              print(f'status={status}', file=file)
-          PY
-          then
-            echo "::warning::The report could not be written, so it holds the run's conclusion alone."
-            echo "### Unit Tests push run, which concluded \`$status\`" > report/unit-tests-report.md
-            echo "status=$status" >> "$GITHUB_OUTPUT"
-          fi
-          cat report/unit-tests-report.md
+        run: python3 util/check_suite_report.py --report report/run-report.md
 
       # The script labels the run it links with GITHUB_WORKFLOW, which names
       # this workflow, so the command sets the relayed run's for that process.
       - name: Post the report to Discussions
+        if: ${{ steps.report.outputs.post == 'true' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ github.token }}
+          WORKFLOW: ${{ github.event.workflow_run.name }}
           TITLE: ${{ steps.report.outputs.title }}
+          MODE: ${{ steps.report.outputs.mode }}
           STATUS: ${{ steps.report.outputs.status }}
           RUN_URL: ${{ github.event.workflow_run.html_url }}
           HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
         run: |
-          GITHUB_WORKFLOW='Unit Tests' python3 util/discussion_report.py --category "$REPORT_CATEGORY" \
-            --title "$TITLE" --report report/unit-tests-report.md \
-            --mode push --status "$STATUS" --run-url "$RUN_URL" --sha "$HEAD_SHA"
+          GITHUB_WORKFLOW="$WORKFLOW" python3 util/discussion_report.py --category "$REPORT_CATEGORY" \
+            --title "$TITLE" --report report/run-report.md \
+            --mode "$MODE" --status "$STATUS" --run-url "$RUN_URL" --sha "$HEAD_SHA"
+

.github/workflows/experimental.yml

@@ -4,8 +4,9 @@ name: Experimental Tests
 # workflow of its own so that nothing waits on its legs (#1732): a
 # `workflow_run` consumer of "Unit Tests" fires only once that run completes,
 # and GraalPy's leg can take the job's whole cap. No `workflow_run` names this
-# workflow, it has no `workflow_call` trigger, and a job cannot `needs:` one in
-# another workflow.
+# workflow but ci-reports.yml's, which posts a finished run's report and holds
+# up nothing (#2030); it has no `workflow_call` trigger, and a job cannot
+# `needs:` one in another workflow.
 #
 # Its check names, `Python <version> (experimental)`, are the ones the legs had
 # in unit-tests.yml; no required context of ruleset 23497679 names one.
@@ -526,7 +527,7 @@ jobs:
           last=${last:-no resource-monitor sample}
           warn() {
             echo "::warning title=${1} on ${LEG//,/%2C}::${2//%/%25} Experimental legs never block (#1660), so this job still succeeds."
-            echo "- ${LEG}: ${1}; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY"
+            echo "- ${LEG}: ${1}; non-blocking, so this job still succeeds." | tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY"
           }
           code=$(value exit) scope=$(value scope) elapsed=$(value elapsed)
           oom=$(value oom_kills) errors=$(value memory_errors) crashed=$(value crashed_workers)
@@ -559,4 +560,30 @@ jobs:
         env:
           LEG: Python ${{ matrix.python-version }} (experimental, non-blocking)
           STEPS: ${{ toJSON(steps) }}
+          JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md
         run: /usr/bin/python3 util/leg_summary.py experimental
+
+      # #2030: the job's summary, for the report ci-reports.yml posts, as in
+      # unit-tests.yml's `test` job. `Containment summary` and `Experimental
+      # summary` copy theirs to job-summary.md; `Run unit tests` does not, as
+      # its tests write summaries of their own, so util/superseded.py's line
+      # for a stopped run stays in the job's summary alone.
+      - name: Name the job summary
+        if: ${{ always() }}
+        continue-on-error: true
+        shell: bash
+        env:
+          JOB_NAME: Python ${{ matrix.python-version }} (experimental)
+        run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi
+
+      - name: Upload the job summary
+        if: ${{ always() }}
+        continue-on-error: true
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: job-summary-${{ github.job }}-${{ strategy.job-index }}
+          path: |
+            ${{ runner.temp }}/job-summary.md
+            ${{ runner.temp }}/job-summary.name
+          if-no-files-found: ignore
+          retention-days: 7

.github/workflows/python-compatibility.yml

@@ -122,4 +122,28 @@ jobs:
         env:
           LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}
           STEPS: ${{ toJSON(steps) }}
+          JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md
         run: /usr/bin/python3 util/leg_summary.py ktlo
+
+      # #2030: the job's summary, for the report ci-reports.yml posts, as in
+      # unit-tests.yml's `test` job: `KTLO summary` copies its lines to
+      # job-summary.md, and these name the job and upload both.
+      - name: Name the job summary
+        if: ${{ always() }}
+        continue-on-error: true
+        shell: bash
+        env:
+          JOB_NAME: Compat Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }}
+        run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi
+
+      - name: Upload the job summary
+        if: ${{ always() }}
+        continue-on-error: true
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: job-summary-${{ github.job }}-${{ strategy.job-index }}
+          path: |
+            ${{ runner.temp }}/job-summary.md
+            ${{ runner.temp }}/job-summary.name
+          if-no-files-found: ignore
+          retention-days: 7