From d1103d7da0da731857c5d910f7afb3bcfb1ebe10 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 22 Oct 2026 22:07:18 -0400 Subject: [PATCH] ci: report Experimental Tests and Python Compatibility to Discussions, with step summaries (#2030) util/check_suite_report.py now writes the check-suite report coverage-comment.yml's discussion job wrote inline, and ci-reports.yml, a workflow_run relay, runs the same steps for Experimental Tests and Python Compatibility, so none of the three gains a write scope. - Push runs: an annotation counts only on a (KTLO) or (experimental) leg, or one titled (KTLO, non-blocking); a passing blocking job's no longer posts. Scheduled runs count every annotation and still post every run. - pip's 'Retrying ... after connection' lines never count, on any leg. - A run whose gated legs all skipped (current or changes.code false) posts nothing. - Each job's summary is copied to job-summary.md (util/_actions.py's JOB_SUMMARY_COPY, or tee), named and uploaded on always(); the report folds each into a
block, trimmed to fit with a link to the run. --- .github/workflows/ci-reports.yml | 97 ++ .github/workflows/coverage-comment.yml | 141 +-- .github/workflows/experimental.yml | 33 +- .github/workflows/python-compatibility.yml | 24 + .github/workflows/unit-tests.yml | 118 ++- docs/source/contributing/workflows.rst | 134 ++- tests/project/test_actions_helper_2047.py | 7 +- tests/project/test_ci_timing.py | 2 + .../test_ktlo_jobs_gate_nothing_1738.py | 6 +- .../project/test_ktlo_legs_stay_green_1672.py | 57 +- .../project/test_python_matrix_tiers_1633.py | 6 +- tests/project/test_resource_monitor_1740.py | 4 +- .../test_sweeps_on_latest_only_1732.py | 14 +- .../test_unit_tests_push_reports_2030.py | 850 ++++++++++++++---- tests/project/test_workflow_scripts_1967.py | 3 +- .../test_workflows_at_a_glance_1732.py | 5 +- util/_actions.py | 90 +- util/check_suite_report.py | 397 ++++++++ 18 files changed, 1589 insertions(+), 399 deletions(-) create mode 100644 .github/workflows/ci-reports.yml create mode 100644 util/check_suite_report.py diff --git a/.github/workflows/ci-reports.yml b/.github/workflows/ci-reports.yml new file mode 100644 index 0000000000..86c6c23e03 --- /dev/null +++ b/.github/workflows/ci-reports.yml @@ -0,0 +1,97 @@ +name: CI Reports + +# #2030: posts the runs of Experimental Tests and Python Compatibility that have +# no pull request to comment on, a push or the schedule, to their threads in the +# Discussions category REPORT_CATEGORY names, as coverage-comment.yml's +# `discussion` job posts Unit Tests'. A `workflow_run` relay, so neither +# workflow's own jobs gain a write scope, and every report is read when its run +# has finished. +# +# `workflow_run` runs this file as it is on the default branch. The job checks +# out the default branch's scripts alone, never the run's head; the run's +# fields reach the scripts as `env:`, and its jobs' summaries, from their +# `job-summary-*` artifacts, are data the report quotes. +on: + workflow_run: + workflows: ["Experimental Tests", "Python Compatibility"] + types: [completed] + +# The job holds its own scopes. +permissions: {} + +# Each relayed run's group is its own: a group cancels a pending run when a +# newer one joins it, which would drop that run's report. +concurrency: + group: ci-reports-${{ github.event.workflow_run.id }} + cancel-in-progress: false + +# One Discussions category holds every report with no pull request to comment +# on, one thread per job. +env: + REPORT_CATEGORY: CI reports + +jobs: + # A push run posts as `push` to `: push runs`, + # and a scheduled one as `scheduled` to `... scheduled runs`; + # util/check_suite_report.py names the thread and the mode. Never a pull + # request's run, whose checks are its report. The steps are + # coverage-comment.yml `discussion`'s, which + # tests/project/test_unit_tests_push_reports_2030.py holds identical. The job + # and its steps are `continue-on-error`; both scripts warn instead of failing. + discussion: + name: Post the run report + if: ${{ github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + continue-on-error: true + permissions: + actions: read + checks: read + contents: read + discussions: write + + steps: + # The default branch, which is what `workflow_run` checks out, and only + # the scripts; nothing from the run's head. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + util/_actions.py + util/check_suite_report.py + util/discussion_report.py + sparse-checkout-cone-mode: false + persist-credentials: false + + # The run's fields are data, in `env:`, where the script reads them. + - name: Write the run report + id: report + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + WORKFLOW: ${{ github.event.workflow_run.name }} + SUITE: ${{ github.event.workflow_run.check_suite_node_id }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + RUN_EVENT: ${{ github.event.workflow_run.event }} + RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + run: python3 util/check_suite_report.py --report report/run-report.md + + # The script labels the run it links with GITHUB_WORKFLOW, which names + # this workflow, so the command sets the relayed run's for that process. + - name: Post the report to Discussions + if: ${{ steps.report.outputs.post == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + WORKFLOW: ${{ github.event.workflow_run.name }} + TITLE: ${{ steps.report.outputs.title }} + MODE: ${{ steps.report.outputs.mode }} + STATUS: ${{ steps.report.outputs.status }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + GITHUB_WORKFLOW="$WORKFLOW" python3 util/discussion_report.py --category "$REPORT_CATEGORY" \ + --title "$TITLE" --report report/run-report.md \ + --mode "$MODE" --status "$STATUS" --run-url "$RUN_URL" --sha "$HEAD_SHA" diff --git a/.github/workflows/coverage-comment.yml b/.github/workflows/coverage-comment.yml index 77275accf8..9ac7268a4f 100644 --- a/.github/workflows/coverage-comment.yml +++ b/.github/workflows/coverage-comment.yml @@ -18,9 +18,12 @@ name: Coverage Comment # # A push run has no pull request to comment on, so the `discussion` job posts it # to its branch's thread in the Discussions category REPORT_CATEGORY names -# (#2030). It reads the run's jobs through the API, not the run's artifacts, -# runs util/discussion_report.py from the default branch's checkout, never the -# run's head, and alone holds `discussions: write`. +# (#2030). It reads the run's jobs through the API and the jobs' summaries from +# their `job-summary-*` artifacts, as data the report quotes, runs +# util/check_suite_report.py and util/discussion_report.py from the default +# branch's checkout, never the run's head, and alone holds `discussions: write`. +# ci-reports.yml's `discussion` job runs the same steps for Experimental Tests +# and Python Compatibility. on: workflow_run: workflows: ["Unit Tests"] @@ -133,13 +136,11 @@ jobs: # #2030: a Unit Tests run on a push, posted as a `push` report to the thread # `unit-tests: push runs`: a run that failed or warned posts, then one - # green run after it. The report is built from the run's check suite, read in - # one GraphQL call rather than from an artifact, and quotes each job name and - # annotation as a code span, so none renders or mentions. A green run reports - # `warning` where a job failed without failing it, or a job that passed holds - # a warning or failure annotation, as a failing KTLO leg does (#1672), other - # than those `EXPECTED` lists as written on every run by design. The job and - # its steps are `continue-on-error`; util/discussion_report.py warns instead. + # green run after it. util/check_suite_report.py writes the report from the + # run's check suite and its jobs' summaries; ci-reports.yml's `discussion` job + # runs the same steps, which tests/project/test_unit_tests_push_reports_2030.py + # holds identical. The job and its steps are `continue-on-error`; both scripts + # warn instead of failing. discussion: name: Post the push run report if: ${{ github.event.workflow_run.event == 'push' }} @@ -147,142 +148,54 @@ jobs: timeout-minutes: 10 continue-on-error: true permissions: + actions: read checks: read contents: read discussions: write steps: # The default branch, which is what `workflow_run` checks out, and only - # the script; nothing from the run's head. + # the scripts; nothing from the run's head. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: sparse-checkout: | + util/_actions.py + util/check_suite_report.py util/discussion_report.py sparse-checkout-cone-mode: false persist-credentials: false + # The run's fields are data, in `env:`, where the script reads them. - name: Write the run report id: report continue-on-error: true env: GH_TOKEN: ${{ github.token }} + WORKFLOW: ${{ github.event.workflow_run.name }} SUITE: ${{ github.event.workflow_run.check_suite_node_id }} + RUN_ID: ${{ github.event.workflow_run.id }} RUN_URL: ${{ github.event.workflow_run.html_url }} + RUN_EVENT: ${{ github.event.workflow_run.event }} RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - run: | - set -u - # The run's conclusion as a --status; anything else, `timed_out` and - # `startup_failure` among them, is a failure. - case "$RUN_CONCLUSION" in - success) status=success ;; - cancelled|stale) status=cancelled ;; - skipped|neutral) status=skipped ;; - *) status=failure ;; - esac - branch=$(printf '%s' "$HEAD_BRANCH" | tr -c 'A-Za-z0-9._/-' '-') - echo "title=unit-tests: ${branch:-unknown} push runs" >> "$GITHUB_OUTPUT" - - # shellcheck disable=SC2016 # `$id` is a GraphQL variable - query='query($id: ID!) { node(id: $id) { ... on CheckSuite { checkRuns(first: 100) { - pageInfo { hasNextPage } - nodes { name conclusion annotations(first: 20) { nodes { annotationLevel title message } } } } } } }' - if ! gh api graphql -f query="$query" -f id="$SUITE" > "$RUNNER_TEMP/jobs.json"; then - echo "::warning::The run's jobs could not be read, so the report names none of them." - : > "$RUNNER_TEMP/jobs.json" - fi - - mkdir -p report - if ! STATUS="$status" python3 - "$RUNNER_TEMP/jobs.json" report/unit-tests-report.md <<'PY' - import json, os, re, sys - status, conclusion = os.environ['STATUS'], os.environ['RUN_CONCLUSION'] - sha, url = os.environ['HEAD_SHA'], os.environ['RUN_URL'] - - def code(text): - """One line of ``text``, at most 200 characters, as a code span, so it renders and mentions nothing.""" - text = ' '.join(str(text).replace('`', "'").split()) - return '`{}`'.format(text if len(text) <= 200 else text[:197] + '...') - - # The annotations a job writes on every run by design, which neither - # count nor are listed: per job, `(level, title prefix)`, where an - # empty prefix matches only an empty title, the runner's own line for - # the step that failed. Matched on the title, not the message, which - # #2041 rewords. Anything else in these jobs, a failed tests step - # among it, still makes a green run `warning`. - # None today: the 3.7 legs that stopped at install went with #2051, and - # pypy3.10's installs since #2040's cryptography cap. - EXPECTED = {} - - def expected(job, note): - """Whether ``note``, an annotation of ``job``, is one :data:`EXPECTED` lists.""" - title = note.get('title') or '' - return any(note.get('annotationLevel') == level and (title.startswith(prefix) if prefix else not title) - for level, prefix in EXPECTED.get(job, ())) - - try: - with open(sys.argv[1], encoding='utf-8') as file: - runs = json.load(file)['data']['node']['checkRuns'] - nodes, more = [node for node in runs['nodes'] if node], runs['pageInfo']['hasNextPage'] - except (OSError, ValueError, LookupError, TypeError): - nodes, more = None, False - failed, warned, by_design = [], [], False - for node in nodes or []: - name, result = node.get('name') or '?', (node.get('conclusion') or 'unknown').lower() - if result not in ('success', 'skipped', 'neutral'): - failed.append(f'{code(name)} ({result})') - elif result == 'success': - flagged = [note for note in (node.get('annotations') or {}).get('nodes') or [] - if note and note.get('annotationLevel') in ('WARNING', 'FAILURE')] - kept = [note for note in flagged if not expected(name, note)] - by_design = by_design or len(kept) < len(flagged) - notes = [note.get('title') or note.get('message') or '' for note in kept] - notes = list(dict.fromkeys(note for note in notes if note.strip()))[:5] - if notes: - warned.append(f'{code(name)}: ' + '; '.join(map(code, notes))) - if status == 'success' and (failed or warned): - status = 'warning' - - run = 'The Unit Tests run' - if re.fullmatch(r'https://[\w.-]+/[\w.-]+/[\w.-]+/actions/runs/\d+', url): - run = f'[Unit Tests run]({url})' - commit = f' for `{sha[:9]}`' if re.fullmatch(r'[0-9a-f]{40}', sha) else '' - lines = ['### Unit Tests push run', '', f'{run}{commit} concluded {code(conclusion or "unknown")}.'] - if failed: - heading = 'Failed without failing the run' if conclusion == 'success' else 'Did not pass' - lines += ['', f'**{heading}:**', '', *(f'- {item}' for item in failed)] - if warned: - lines += ['', '**Warned, in jobs that passed:**', '', *(f'- {item}' for item in warned)] - if nodes is None: - lines += ['', "The run's jobs could not be read, so none is named here."] - elif more: - lines += ['', 'Only the first 100 jobs were read.'] - elif not failed and not warned: - lines += ['', 'Every job passed, with no warning but the by-design ones.' if by_design - else 'Every job passed, with no warning.'] - with open(sys.argv[2], 'w', encoding='utf-8') as file: - print('\n'.join(lines), file=file) - with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as file: - print(f'status={status}', file=file) - PY - then - echo "::warning::The report could not be written, so it holds the run's conclusion alone." - echo "### Unit Tests push run, which concluded \`$status\`" > report/unit-tests-report.md - echo "status=$status" >> "$GITHUB_OUTPUT" - fi - cat report/unit-tests-report.md + run: python3 util/check_suite_report.py --report report/run-report.md # The script labels the run it links with GITHUB_WORKFLOW, which names # this workflow, so the command sets the relayed run's for that process. - name: Post the report to Discussions + if: ${{ steps.report.outputs.post == 'true' }} continue-on-error: true env: GH_TOKEN: ${{ github.token }} + WORKFLOW: ${{ github.event.workflow_run.name }} TITLE: ${{ steps.report.outputs.title }} + MODE: ${{ steps.report.outputs.mode }} STATUS: ${{ steps.report.outputs.status }} RUN_URL: ${{ github.event.workflow_run.html_url }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} run: | - GITHUB_WORKFLOW='Unit Tests' python3 util/discussion_report.py --category "$REPORT_CATEGORY" \ - --title "$TITLE" --report report/unit-tests-report.md \ - --mode push --status "$STATUS" --run-url "$RUN_URL" --sha "$HEAD_SHA" + GITHUB_WORKFLOW="$WORKFLOW" python3 util/discussion_report.py --category "$REPORT_CATEGORY" \ + --title "$TITLE" --report report/run-report.md \ + --mode "$MODE" --status "$STATUS" --run-url "$RUN_URL" --sha "$HEAD_SHA" + diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index a6081fd6db..0b8c44a352 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -4,8 +4,9 @@ name: Experimental Tests # workflow of its own so that nothing waits on its legs (#1732): a # `workflow_run` consumer of "Unit Tests" fires only once that run completes, # and GraalPy's leg can take the job's whole cap. No `workflow_run` names this -# workflow, it has no `workflow_call` trigger, and a job cannot `needs:` one in -# another workflow. +# workflow but ci-reports.yml's, which posts a finished run's report and holds +# up nothing (#2030); it has no `workflow_call` trigger, and a job cannot +# `needs:` one in another workflow. # # Its check names, `Python (experimental)`, are the ones the legs had # in unit-tests.yml; no required context of ruleset 23497679 names one. @@ -526,7 +527,7 @@ jobs: last=${last:-no resource-monitor sample} warn() { echo "::warning title=${1} on ${LEG//,/%2C}::${2//%/%25} Experimental legs never block (#1660), so this job still succeeds." - echo "- ${LEG}: ${1}; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + echo "- ${LEG}: ${1}; non-blocking, so this job still succeeds." | tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY" } code=$(value exit) scope=$(value scope) elapsed=$(value elapsed) oom=$(value oom_kills) errors=$(value memory_errors) crashed=$(value crashed_workers) @@ -559,4 +560,30 @@ jobs: env: LEG: Python ${{ matrix.python-version }} (experimental, non-blocking) STEPS: ${{ toJSON(steps) }} + JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md run: /usr/bin/python3 util/leg_summary.py experimental + + # #2030: the job's summary, for the report ci-reports.yml posts, as in + # unit-tests.yml's `test` job. `Containment summary` and `Experimental + # summary` copy theirs to job-summary.md; `Run unit tests` does not, as + # its tests write summaries of their own, so util/superseded.py's line + # for a stopped run stays in the job's summary alone. + - name: Name the job summary + if: ${{ always() }} + continue-on-error: true + shell: bash + env: + JOB_NAME: Python ${{ matrix.python-version }} (experimental) + run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi + + - name: Upload the job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: job-summary-${{ github.job }}-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/job-summary.md + ${{ runner.temp }}/job-summary.name + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/python-compatibility.yml b/.github/workflows/python-compatibility.yml index 6428926fb6..19619eadb8 100644 --- a/.github/workflows/python-compatibility.yml +++ b/.github/workflows/python-compatibility.yml @@ -122,4 +122,28 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} + JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md run: /usr/bin/python3 util/leg_summary.py ktlo + + # #2030: the job's summary, for the report ci-reports.yml posts, as in + # unit-tests.yml's `test` job: `KTLO summary` copies its lines to + # job-summary.md, and these name the job and upload both. + - name: Name the job summary + if: ${{ always() }} + continue-on-error: true + shell: bash + env: + JOB_NAME: Compat Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi + + - name: Upload the job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: job-summary-${{ github.job }}-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/job-summary.md + ${{ runner.temp }}/job-summary.name + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index fb2933f81b..7a161319ba 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -231,7 +231,7 @@ jobs: # measured process saved its data (#1387). coverage-comment.yml posts the # PR comment from the artifact, since callers of this reusable workflow # grant only `contents: read`. `!cancelled()` so a failing test still - # reports what it measured. + # reports what it measured. #2030: its summary goes to the job's copy too. - name: Report coverage if: ${{ !cancelled() && matrix.coverage }} shell: bash @@ -239,6 +239,7 @@ jobs: SHA: ${{ github.sha }} PYTHON: ${{ matrix.python-version }} TESTS_OUTCOME: ${{ steps.tests.outcome }} + JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md run: python util/coverage_report.py # #1423: this log is the only record of a measured process that saved no @@ -281,12 +282,12 @@ jobs: path: ${{ runner.temp }}/junit*.xml retention-days: 14 - # #1672: the last step of every KTLO matrix job, util/leg_summary.py - # (#1967). Each step above that can fail on a KTLO leg has an `id` and - # `continue-on-error: ${{ matrix.ktlo == true }}`, so its failure leaves - # the job at success; this step makes it visible again, one warning and - # one job-summary line per failed step, from each `outcome` in - # `toJSON(steps)`. It runs on the image's /usr/bin/python3, as the leg's + # #1672: the last step of every KTLO matrix job but the job summary's + # two, util/leg_summary.py (#1967). Each step above that can fail on a + # KTLO leg has an `id` and `continue-on-error: ${{ matrix.ktlo == true }}`, + # so its failure leaves the job at success; this step makes it visible + # again, one warning and one job-summary line per failed step, from each + # `outcome` in `toJSON(steps)`. It runs on the image's /usr/bin/python3, as the leg's # may have failed to set up. The runner's own `Process completed with # exit code N.` error for a failed `run:` step cannot be removed, but # does not change the job's conclusion. @@ -297,8 +298,35 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} + JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md run: /usr/bin/python3 util/leg_summary.py ktlo + # #2030: the job's summary, for the push report util/check_suite_report.py + # writes, as the API serves none. Each step above that writes the summary + # copies it to job-summary.md, through util/_actions.py's + # JOB_SUMMARY_COPY or a `tee`; this names the job beside it, and the + # upload takes both. On `always()`, so a failed or timed-out leg's + # summary is kept too, and masked, as a diagnostic upload is. + - name: Name the job summary + if: ${{ always() }} + continue-on-error: true + shell: bash + env: + JOB_NAME: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi + + - name: Upload the job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: job-summary-${{ github.job }}-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/job-summary.md + ${{ runner.temp }}/job-summary.name + if-no-files-found: ignore + retention-days: 7 + # The KTLO tier (#1633): end-of-life versions that still build and test but # never block. #1738, the owner's option A: their legs are this job's, not # `test`'s, and neither `required-checks` nor `unittest-ordering` `needs:` @@ -457,8 +485,30 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} + JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md run: /usr/bin/python3 util/leg_summary.py ktlo + # #2030: see the `test` job's steps of the same names. + - name: Name the job summary + if: ${{ always() }} + continue-on-error: true + shell: bash + env: + JOB_NAME: Integration Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi + + - name: Upload the job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: job-summary-${{ github.job }}-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/job-summary.md + ${{ runner.temp }}/job-summary.name + if-no-files-found: ignore + retention-days: 7 + # `integration`'s KTLO legs, apart for the reasons on `test-ktlo` (#1738). integration-ktlo: name: Integration Python ${{ matrix.python-version }} (KTLO) @@ -716,7 +766,7 @@ jobs: exit 1 fi echo "::notice title=$ENGINE is not installable on Python $PY (expected)::pip exited $status, matching this cell's recorded expectation ('not-installable'), and the output does not match a transient-failure pattern. No tests run for this cell -- this is a deliberate skip, not a pass." - echo "| $PY | $ENGINE | $EXPECT | not installable (expected) |" >> "$GITHUB_STEP_SUMMARY" + echo "| $PY | $ENGINE | $EXPECT | not installable (expected) |" | tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY" exit 0 fi @@ -741,14 +791,14 @@ jobs: echo "::${ANNOTATION_LEVEL} title=$ENGINE tests failed on Python $PY::See the 'Python $PY ($ENGINE)' group in this step's log." exit 1 fi - echo "| $PY | $ENGINE | $EXPECT | passed |" >> "$GITHUB_STEP_SUMMARY" + echo "| $PY | $ENGINE | $EXPECT | passed |" | tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY" CELL { echo "### Engine cells: $ENGINE" echo echo "| Python | Engine | Expect | Result |" echo "| --- | --- | --- | --- |" - } >> "$GITHUB_STEP_SUMMARY" + } | tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY" # See the `test` job above for why this step exists. - name: Report available parallelism @@ -770,7 +820,7 @@ jobs: echo "::group::Python $py ($ENGINE)" if ! bash "$RUNNER_TEMP/engine-cell.sh" "$py"; then failed+=("$py") - echo "| $py | $ENGINE | | FAILED |" >> "$GITHUB_STEP_SUMMARY" + echo "| $py | $ENGINE | | FAILED |" | tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY" fi echo "::endgroup::" done @@ -795,7 +845,7 @@ jobs: echo "::group::Python $py ($ENGINE)" if ! bash "$RUNNER_TEMP/engine-cell.sh" "$py"; then failed+=("$py") - echo "| $py | $ENGINE | | FAILED (KTLO, non-blocking) |" >> "$GITHUB_STEP_SUMMARY" + echo "| $py | $ENGINE | | FAILED (KTLO, non-blocking) |" | tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY" fi echo "::endgroup::" done @@ -804,6 +854,28 @@ jobs: exit 1 fi + # #2030: see the `test` job's steps of the same names; the steps above + # copy the cells' table themselves, with `tee`. + - name: Name the job summary + if: ${{ always() }} + continue-on-error: true + shell: bash + env: + JOB_NAME: Engines (${{ matrix.engine }}) + run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi + + - name: Upload the job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: job-summary-${{ github.job }}-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/job-summary.md + ${{ runner.temp }}/job-summary.name + if-no-files-found: ignore + retention-days: 7 + # #751's ruling: try building upstream PyPCAP in CI, and drop it if CI is # not a good fit. If a clean run of this job's install step goes red -- not # a flake -- the fix is deleting this job and returning HAS_PYPCAP to @@ -979,8 +1051,30 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} + JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md run: /usr/bin/python3 util/leg_summary.py ktlo + # #2030: see the `test` job's steps of the same names. + - name: Name the job summary + if: ${{ always() }} + continue-on-error: true + shell: bash + env: + JOB_NAME: PyPCAP/PyPCAPFile parity Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi + + - name: Upload the job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: job-summary-${{ github.job }}-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/job-summary.md + ${{ runner.temp }}/job-summary.name + if-no-files-found: ignore + retention-days: 7 + # `pypcap-parity`'s KTLO legs, apart for the reasons on `test-ktlo` (#1738). # Same apt set and cache key; `pypcap-parity`'s supported leg saves it. pypcap-parity-ktlo: diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 964edf048c..7f2d3f9b3a 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -107,6 +107,13 @@ At a Glance - -- - **Unit Tests** - -- + * - CI Reports + - :file:`ci-reports.yml` + - -- + - -- + - -- + - **Experimental Tests**, **Python Compatibility** + - -- * - CI Timing - :file:`ci-timing.yml` - -- @@ -124,8 +131,8 @@ At a Glance .. note:: - Only Coverage Comment fires *exclusively* on ``workflow_run``: it has - nothing to do without a completed Unit Tests run. Each of the others + Only Coverage Comment and CI Reports fire *exclusively* on ``workflow_run``: + neither has anything to do without a completed run to report. Each of the others that carry one pairs it with a direct trigger of its own -- a Saturday ``schedule`` for Vendor Update, Conda Update and GitHub Pages, a ``v*`` tag push for Create Release -- for the path where no upstream run exists yet to @@ -220,12 +227,17 @@ than starting a new one. CH["Cron Head
cron-head.yml"] CC["Coverage Comment
coverage-comment.yml"] CR["Create Release
create-release.yml"] + EX["Experimental Tests
experimental.yml"] + PC["Python Compatibility
python-compatibility.yml"] + RP["CI Reports
ci-reports.yml"] UT ==>|workflow_run: completed| VU UT ==>|workflow_run: completed| CU UT ==>|workflow_run: completed| GP VU ==>|workflow_run: completed| CR UT ==>|workflow_run: completed| CC + EX ==>|workflow_run: completed| RP + PC ==>|workflow_run: completed| RP VU -.-> CH CU -.-> CH @@ -267,13 +279,17 @@ re-derives the list: **Create Release** fires on completion of **Vendor Update**. * ``.github/workflows/coverage-comment.yml``, its ``workflow_run`` trigger -- **Coverage Comment** fires on completion of **Unit Tests**. +* ``.github/workflows/ci-reports.yml``, its ``workflow_run`` trigger -- + **CI Reports** fires on completion of **Experimental Tests** and of **Python + Compatibility**. ``workflow_run`` fires for *every* completion of the named workflow -- success, failure, or skipped -- and regardless of what triggered it, including a pull request's own run of Unit Tests. Every downstream workflow therefore re-checks ``github.event.workflow_run.event`` and ``.head_branch`` (or, for Create Release and Coverage Comment, -``.conclusion``) itself before doing anything with an outward effect; none of +``.conclusion``; for CI Reports, the event alone) itself before doing anything +with an outward effect; none of the filtering happens in the ``on:`` block. Those checks say nothing about *which* commit was verified. A ``workflow_run`` @@ -769,8 +785,9 @@ with supported legs. Every job holding KTLO legs works the same way: and ``required-checks`` read ``success``. A step that needs an earlier one's work waits on that step's ``outcome``: a leg that cannot install stops there, and a ``pypi`` leg stops at its first failure. -* The last step, ``KTLO summary``, runs on KTLO legs only and reports each failed - ``outcome`` as a warning and a job-summary line, through +* The last step but the job summary's (see `CI Reports`_), ``KTLO summary``, + runs on KTLO legs only and reports each failed ``outcome`` as a warning and a + job-summary line, through :file:`util/leg_summary.py` (see `Workflow Scripts (#1967)`_). A step's own annotations take ``ANNOTATION_LEVEL``: ``warning`` on a KTLO leg, ``error`` on a supported one. ``engine-tests``' KTLO cells, a step of their own with ``continue-on-error: true``, @@ -834,7 +851,8 @@ context. They were legs of ``test`` until `#1732 `__, which moved them to a workflow of their own, by the owner's ruling that nothing waits on them. **That workflow reports separately and holds up nothing**: a job cannot ``needs:`` one in -another workflow, nothing calls it, and no ``workflow_run`` names it, so the Unit +another workflow, nothing calls it, and no ``workflow_run`` names it but CI +Reports', which posts its finished runs (see `CI Reports`_), so the Unit Tests run completes without it, and with it ``Required checks passed``, ``unittest-ordering`` and the workflows that wait on Unit Tests completing (``coverage-comment.yml``, ``deploy-pages.yml``, ``cron-vendor.yml`` and @@ -884,7 +902,7 @@ asks with ``If-None-Match``, as a ``304`` answer costs no rate limit. The legs never block, by the KTLO mechanics above: every step that can fail has ``continue-on-error: true``, their annotations are warnings, and ``Experimental -summary``, the job's last step, reports each failed step. A failure no step +summary``, the job's last step but the job summary's, reports each failed step. A failure no step contains, such as a lost runner, fails the job, and the job's own ``continue-on-error: true`` keeps it from failing the run: it "prevents a workflow run from failing when a job fails" (`Workflow syntax @@ -995,7 +1013,7 @@ down`` lines. From it and the monitor's last sample, ``Containment summary`` war minutes``, ``tests raised MemoryError`` or ``tests lost an xdist worker``, each a warning of its own. A timeout is exit 124, or 137 with no OOM kill after 110 minutes: ``--kill-after``'s SIGKILL to a teardown SIGINT did not end, where an -OOM kill of pytest exits 137 too. ``Experimental summary``, the job's last step and the +OOM kill of pytest exits 137 too. ``Experimental summary``, the job's last step but the job summary's and the KTLO summary's script, still warns of every failed step, ``tests`` included. The 2 GiB, ``-n 3`` and 12G are estimates for a 16 GB, 4-CPU runner until a run shows them. At 3 workers rather than 4, GraalPy's tests should take about 117-151 minutes (an inference @@ -1219,8 +1237,13 @@ does; each has tests under :file:`tests/project/` that run it. * - :file:`util/discussion_report.py` - ``Post the report to Discussions`` in the ``discussion`` job of ``ci-timing.yml``, ``cron-vendor.yml``, ``cron-conda.yml``, - ``cron-python-versions.yml``, ``lint.yml`` and ``deploy-pages.yml`` (see - `CI Reports`_). + ``cron-python-versions.yml``, ``lint.yml``, ``deploy-pages.yml``, + ``codeql-analysis.yml``, ``coverage-comment.yml`` and ``ci-reports.yml`` + (see `CI Reports`_). + * - :file:`util/check_suite_report.py` + - ``Write the run report`` in the ``discussion`` job of + ``coverage-comment.yml``, for Unit Tests, and of ``ci-reports.yml``, for + Experimental Tests and Python Compatibility (see `CI Reports`_). * - :file:`util/apt_packages.py` - The apt steps of ``unit-tests.yml``'s ``engine-tests`` and ``pypcap-parity`` (and so its KTLO twin): the plan, seed, install and collect steps, as @@ -1353,35 +1376,68 @@ a Python Versions run failed by its audit. The report steps of ``vendor-update`` ``conda-update`` and ``detect`` are ``continue-on-error`` on ``!cancelled()``, and come last in each job. -Unit Tests push runs post as ``push`` to ``unit-tests: push runs``, so -``main``'s thread is ``unit-tests: main push runs``. ``unit-tests.yml`` is a -reusable workflow whose callers grant ``contents: read``, so the post comes from -``coverage-comment.yml``'s ``workflow_run`` relay instead: its ``discussion`` job -takes a run whose event is ``push``, and ``comment`` keeps a pull request's. Each -job holds its own scopes, ``discussion`` alone ``discussions: write``, with -``contents: read`` for a checkout of the default branch's script, never the run's -head, and ``checks: read`` for the run's check suite. Its ``Write the run report`` -step reads that suite in a single GraphQL call rather than any artifact, and -renders every name and annotation it quotes as a code span. The report links the -run, names its commit and conclusion, the jobs that did not pass, and the warning -and failure annotations of the jobs that did. The conclusion becomes the -``--status``: ``success``, ``cancelled`` (and ``stale``), ``skipped`` (and -``neutral``), and ``failure`` for anything else, ``timed_out`` and -``startup_failure`` among them. A green run reports ``warning`` where a job failed -without failing it, or a job that passed holds a warning or failure annotation, as -a KTLO leg whose tests fail does (`#1672 -`__). The exception is the -step's ``EXPECTED`` list: the annotations a job writes on every run by design, -today the install failures of the KTLO legs that cannot install, which neither -count nor appear in the report. Each entry is a job, a level and a title prefix, -an empty prefix matching only an empty title, so a reworded message still -matches, and anything else those jobs write still counts. -:file:`tests/project/test_unit_tests_push_reports_2030.py` pins the list, so a -leg that starts to install, or a new by-design annotation, changes it and the -pin together. The script labels the -run it links with ``GITHUB_WORKFLOW``, so the post step sets ``Unit Tests`` for -that one command. A push relay's concurrency group is its own run's, since a -group cancels a pending run when a newer one joins it. +The test workflows report through a ``workflow_run`` relay, so none of their own +jobs gains a write scope: ``unit-tests.yml`` is a reusable workflow whose callers +grant ``contents: read``. Unit Tests' push runs go through +``coverage-comment.yml``, whose ``discussion`` job takes a run whose event is +``push`` while ``comment`` keeps a pull request's. Experimental Tests' and Python +Compatibility's go through ``ci-reports.yml``, whose ``discussion`` job takes a +``push`` or ``schedule`` run and never a pull request's. Both ``discussion`` jobs +run the same steps, and alone hold ``discussions: write``, with ``contents: read`` +for a checkout of the default branch's scripts, never the run's head, +``checks: read`` for the run's check suite and ``actions: read`` for its +artifacts. A push run posts as ``push`` to ``: push +runs``, so ``main``'s Unit Tests thread is ``unit-tests: main push runs``; a +scheduled one posts as ``scheduled`` to ``: scheduled +runs``. Each relayed run's concurrency group is its own, since a group cancels a +pending run when a newer one joins it. + +``Write the run report`` runs :file:`util/check_suite_report.py`, which reads the +run's check suite in a single GraphQL call and renders every name and annotation +it quotes as a code span. The report links the run, names its commit and +conclusion, the jobs that did not pass, and the warning and failure annotations +that count. The conclusion becomes the ``--status``: ``success``, ``cancelled`` +(and ``stale``), ``skipped`` (and ``neutral``), and ``failure`` for anything else, +``timed_out`` and ``startup_failure`` among them. A green run reports ``warning`` +where a job failed without failing it, or where an annotation counts: + +* On a push run, by the owner's ruling, an annotation counts only on a + non-blocking leg, whose steps are forced to succeed: a job named ``(KTLO)`` or + ``(experimental)``, as a KTLO leg whose tests fail warns (`#1672 + `__), or an annotation + titled ``(KTLO, non-blocking)``, as ``engine-tests``' KTLO cells write. A passing + blocking job's annotations neither count nor appear, unless the run failed. +* On any other run, which posts every time, every annotation on a passing job + counts. +* pip's ``Retrying ... after connection`` lines never count, on any leg, and + neither do the annotations a workflow's ``expected`` list in the script holds as + written on every run by design. Each entry is a job, a level and a title + prefix, an empty prefix matching only an empty title, so a reworded message + still matches. Every list is empty today, and + :file:`tests/project/test_unit_tests_push_reports_2030.py` pins them. + +A run whose legs all skipped tested nothing, as when ``current`` found its commit +superseded or ``changes`` found no code changed, so where it did not fail it +posts nothing, rather than a "back to green" after a bad report. The legs are the +jobs those gates skip, which the script names by a pattern per workflow and the +test file holds to the jobs' ``if:``. + +The report ends in each job's summary, one ``
`` block a job, in the check +suite's order. GitHub serves no job summary through its API, so every step that +writes one copies it to the job's ``job-summary.md``: the scripts through +:file:`util/_actions.py`, where the step sets ``JOB_SUMMARY_COPY``, and the shell +with ``tee``. No step that runs the tests sets it, since a test writing the +summary through the helper would write into the copy. The job then ends in ``Name +the job summary``, which writes the job's name beside it, and ``Upload the job +summary``, which uploads both as the ``job-summary--`` artifact (7 +days) for the report to download; both run on ``always()`` and are +``continue-on-error``. The summaries share the room a comment leaves, smallest first, +and one cut to fit links the run. ``Run unit tests`` in ``experimental.yml`` does +not copy its summary, so the line :file:`util/superseded.py` writes there for a +stopped run stays in the job's summary alone. The post step runs only where the +report step's ``post`` output is ``true``, and sets ``GITHUB_WORKFLOW`` to the +relayed run's workflow, which :file:`util/discussion_report.py` labels the run +with. ``codeql-analysis.yml`` posts to ``codeql-analysis: CodeQL report``, as ``scheduled`` on its ``schedule`` and ``push`` on a push to ``main``, and never on diff --git a/tests/project/test_actions_helper_2047.py b/tests/project/test_actions_helper_2047.py index 57123f8e0f..10b57c9c2c 100644 --- a/tests/project/test_actions_helper_2047.py +++ b/tests/project/test_actions_helper_2047.py @@ -40,9 +40,9 @@ ROOT = pathlib.Path(__file__).resolve().parents[2] HELPER = ROOT / 'util' / '_actions.py' #: The scripts that import the helper. -SCRIPTS = ('apt_packages', 'ci_timing', 'classify_changes', 'codeql_report', 'coverage_report', 'leg_summary', - 'pip_install', 'regenerate_samples', 'release_tags', 'release_upload', 'update_python_versions', - 'vendor_code_changed', 'vendor_test_gate') +SCRIPTS = ('apt_packages', 'check_suite_report', 'ci_timing', 'classify_changes', 'codeql_report', + 'coverage_report', 'leg_summary', 'pip_install', 'regenerate_samples', 'release_tags', 'release_upload', + 'update_python_versions', 'vendor_code_changed', 'vendor_test_gate') #: The top-level modules the helper may import, each in Python 3.7's standard #: library: the running interpreter's :data:`sys.stdlib_module_names` would #: pass ``tomllib``, which 3.7 lacks. @@ -50,6 +50,7 @@ #: A trivial run of each script, ``(arguments, environment)``, that gets past its imports. RUNS = { 'apt_packages': (['--help'], {}), + 'check_suite_report': (['--help'], {}), 'ci_timing': (['--help'], {}), 'classify_changes': ([], {'OWN_PR': 'false'}), 'codeql_report': (['--help'], {}), diff --git a/tests/project/test_ci_timing.py b/tests/project/test_ci_timing.py index 6f93739a26..03ca3d163a 100644 --- a/tests/project/test_ci_timing.py +++ b/tests/project/test_ci_timing.py @@ -1138,6 +1138,8 @@ def test_artifact_names_are_unique_in_a_run(self) -> 'None': for pin in _workflows.PINNED.finditer(self.text) if pin['action'] == 'actions/upload-artifact') if found is not None] self.assertGreaterEqual(len(names), 5) + # #2030: a name holding the job's id is its job's alone, however many jobs write it. + names = [name for name in names if '${{ github.job }}' not in name] self.assertEqual(len(names), len(set(names)), names) def test_both_pytest_runs_write_the_file_the_upload_reads(self) -> 'None': diff --git a/tests/project/test_ktlo_jobs_gate_nothing_1738.py b/tests/project/test_ktlo_jobs_gate_nothing_1738.py index d1107b876b..1e1b25a3df 100644 --- a/tests/project/test_ktlo_jobs_gate_nothing_1738.py +++ b/tests/project/test_ktlo_jobs_gate_nothing_1738.py @@ -158,8 +158,10 @@ def test_each_runs_its_twins_steps_on_ktlo_legs_only(self) -> None: self.assertEqual((spec['runs-on'], spec['timeout-minutes']), (gate['runs-on'], gate['timeout-minutes'])) self.assertEqual(spec['env']['ANNOTATION_LEVEL'], gate['env']['ANNOTATION_LEVEL']) self.assertTrue(all(leg.get('ktlo') is True for leg in matrix_legs(twin))) - self.assertEqual(spec['steps'][-1]['name'], 'KTLO summary') - self.assertEqual(spec['steps'][-1]['continue-on-error'], KTLO_GUARD) + # #2030: last but the two that name and upload the job's summary. + self.assertEqual([step['name'] for step in spec['steps'][-3:]], + ['KTLO summary', 'Name the job summary', 'Upload the job summary']) + self.assertEqual(spec['steps'][-3]['continue-on-error'], KTLO_GUARD) def test_each_has_the_job_level_backstop(self) -> None: for twin in TWINS.values(): diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index 9fd8a8570d..c27a969514 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -11,7 +11,9 @@ leg carries ``continue-on-error: ${{ matrix.ktlo == true }}`` and an ``id``; a step that needs an earlier one waits on that step's ``outcome``; the last step, ``KTLO summary``, reports each failed ``outcome`` as a warning and a job-summary -line; and a step's own annotations take the job's ``ANNOTATION_LEVEL``. +line; and a step's own annotations take the job's ``ANNOTATION_LEVEL``. #2030: +only the two steps that name and upload the job's copy of its summary, +:data:`JOB_SUMMARY_STEPS`, follow a summary, masked on every leg. :func:`run_leg` steps through a job the way the runner does -- the implicit ``success()`` GitHub prepends to an ``if:`` without a status function, and a @@ -121,6 +123,12 @@ 'apt-plan': {'key': 'apt-debs-ImageOS-X64-set-0123456789abcdef', 'restore-prefix': 'apt-debs-ImageOS-X64-set-'}, 'apt-cache': {'cache-matched-key': 'apt-debs-ImageOS-X64-set-0123456789abcdef', 'cache-hit': 'false'}, } +#: #2030: the steps after a summary, on ``always()`` and masked on every leg, that +#: name and upload the job's copy of its summary for util/check_suite_report.py. +#: They end every job with a KTLO or experimental leg but :data:`INLINE_SUMMARY`'s. +JOB_SUMMARY_STEPS = ('Name the job summary', 'Upload the job summary') +#: The ``env:`` entry by which a summary step copies its lines there too (#2030). +SUMMARY_COPY = {'JOB_SUMMARY_COPY': '${{ runner.temp }}/job-summary.md'} SUMMARY = 'KTLO summary' SUMMARY_IF = '${{ !cancelled() && matrix.ktlo == true }}' SUMMARY_ENV = {'LEG': "Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}", @@ -177,6 +185,16 @@ def label(step: 'dict[str, Any]') -> 'str': return step.get('name') or step.get('uses') +def summary_tail(where: 'tuple[str, str]') -> 'tuple[str, ...]': + """The steps that follow the summary of the job ``where``, ``(workflow, job)`` (#2030).""" + return () if where == INLINE_SUMMARY else JOB_SUMMARY_STEPS + + +def summary_env(where: 'tuple[str, str]', env: 'dict[str, str]') -> 'dict[str, str]': + """The summary step's ``env:`` for the job ``where``, which copies its lines unless inline (#2030).""" + return dict(env) if where == INLINE_SUMMARY else dict(env, **SUMMARY_COPY) + + def summary_step(spec: 'dict[str, Any]') -> 'dict[str, Any]': """The job's ``KTLO summary`` step.""" found = [step for step in spec['steps'] if step.get('name') == SUMMARY] @@ -411,25 +429,39 @@ def test_every_failure_it_masks_reaches_the_summary(self) -> None: for name, job_id in KTLO_JOBS: spec = job(name, job_id) summary = summary_step(spec) + position = spec['steps'].index(summary) + tail = summary_tail((name, job_id)) with self.subTest(workflow=name, job=job_id): - self.assertIs(spec['steps'][-1], summary) + self.assertEqual(position, len(spec['steps']) - 1 - len(tail)) self.assertEqual(summary.get('if'), SUMMARY_IF) self.assertEqual(summary.get('continue-on-error'), KTLO_GUARD) self.assertEqual(summary.get('shell'), 'bash') - self.assertEqual(summary.get('env'), SUMMARY_ENV) + self.assertEqual(summary.get('env'), summary_env((name, job_id), SUMMARY_ENV)) if (name, job_id) != INLINE_SUMMARY: self.assertEqual(summary['run'], SUMMARY_RUN) blocking = BLOCKS_EVERY_LEG.get((name, job_id), ()) for leg in filter(is_ktlo, legs(name, job_id)): for index, _, record in scenarios(name, job_id, spec, leg): step = spec['steps'][index] - if step is summary or label(step) in blocking: + if step is summary or label(step) in blocking or label(step) in tail: continue with self.subTest(workflow=name, job=job_id, python=leg['python-version'], step=label(step)): - self.assertNotEqual(record[-1].outcome, 'skipped') + self.assertNotEqual(record[position].outcome, 'skipped') if step.get('continue-on-error') is not True: self.assertIn('id', step) + def test_only_the_job_summary_steps_follow_the_summary(self) -> None: + """#2030: they run whatever came before, ``always()``, and nothing they do fails a leg.""" + for where in KTLO_JOBS + (EXPERIMENTAL_JOB,): + spec = job(*where) + tail = summary_tail(where) + steps = spec['steps'][len(spec['steps']) - len(tail):] + with self.subTest(workflow=where[0], job=where[1]): + self.assertEqual([label(step) for step in steps], list(tail)) + for step in steps: + self.assertEqual(step.get('if'), '${{ always() }}') + self.assertIs(step.get('continue-on-error'), True) + class TestAKTLOFailureIsAWarning(unittest.TestCase): """(b) Nothing a KTLO leg runs writes an error, and the summary writes warnings.""" @@ -612,7 +644,7 @@ def test_every_failure_still_fails_a_supported_leg(self) -> None: by_id = {step['id']: step for step in spec['steps'] if 'id' in step} masked = [label(step) for step in spec['steps'] if step.get('continue-on-error') is True] with self.subTest(workflow=name, job=job_id): - self.assertEqual(masked, list(MASKED_ON_EVERY_LEG.get((name, job_id), ()))) + self.assertEqual(masked, [*MASKED_ON_EVERY_LEG.get((name, job_id), ()), *summary_tail((name, job_id))]) for leg in (leg for leg in legs(name, job_id) if not is_nonblocking(leg)): clean, record = run_leg(name, job_id, spec, leg) with self.subTest(workflow=name, job=job_id, python=leg['python-version']): @@ -631,11 +663,12 @@ def test_no_summary_runs_on_a_supported_leg(self) -> None: """``test`` has one, ``KTLO summary``; ``Experimental summary`` left it with its legs (#1732).""" spec = job('unit-tests.yml', 'test') self.assertNotIn(EXPERIMENTAL_SUMMARY, [label(step) for step in spec['steps']]) - self.assertIs(spec['steps'][-1], summary_step(spec)) + position = len(spec['steps']) - 1 - len(JOB_SUMMARY_STEPS) + self.assertIs(spec['steps'][position], summary_step(spec)) for leg in (leg for leg in legs('unit-tests.yml', 'test') if not is_nonblocking(leg)): for _, _, record in scenarios('unit-tests.yml', 'test', spec, leg): with self.subTest(python=leg['python-version']): - self.assertEqual(record[-1].outcome, 'skipped') + self.assertEqual(record[position].outcome, 'skipped') def test_a_supported_engine_cell_still_fails_its_job_as_an_error(self) -> None: spec = job('unit-tests.yml', 'engine-tests') @@ -671,17 +704,17 @@ def test_no_step_can_fail_an_experimental_leg(self) -> None: self.assertEqual(conclusion, 'success') def test_every_failure_it_masks_reaches_the_experimental_summary(self) -> None: - """It runs after any failure on an experimental leg, as the job's last step.""" - self.assertIs(self.spec['steps'][-1], self.summary) + """It runs after any failure on an experimental leg, as the job's last step but the job summary's.""" + self.assertEqual(self.position, len(self.spec['steps']) - 1 - len(JOB_SUMMARY_STEPS)) self.assertNotIn(SUMMARY, [label(step) for step in self.spec['steps']]) self.assertEqual(self.summary.get('if'), EXPERIMENTAL_SUMMARY_IF) self.assertEqual(self.summary.get('continue-on-error'), EXPERIMENTAL_GUARD) self.assertEqual(self.summary.get('shell'), 'bash') - self.assertEqual(self.summary.get('env'), EXPERIMENTAL_SUMMARY_ENV) + self.assertEqual(self.summary.get('env'), summary_env(EXPERIMENTAL_JOB, EXPERIMENTAL_SUMMARY_ENV)) for leg in self.legs: for index, _, record in scenarios(*EXPERIMENTAL_JOB, self.spec, leg): step = self.spec['steps'][index] - if step is self.summary: + if step is self.summary or label(step) in JOB_SUMMARY_STEPS: continue with self.subTest(python=leg['python-version'], step=label(step)): self.assertNotEqual(record[self.position].outcome, 'skipped') diff --git a/tests/project/test_python_matrix_tiers_1633.py b/tests/project/test_python_matrix_tiers_1633.py index 0d7ba9caa8..2fd47b9749 100644 --- a/tests/project/test_python_matrix_tiers_1633.py +++ b/tests/project/test_python_matrix_tiers_1633.py @@ -632,7 +632,8 @@ def test_a_leg_that_cannot_install_stops_there(self) -> None: ``outcome``, not ``conclusion``: ``continue-on-error`` turns a failed step's conclusion into ``success``, so only its outcome still says it failed. #1672 brought Compat and ``pypcap-parity`` in; ``KTLO summary`` is the one step - that has to run after a failed install, to report it. + that has to run after a failed install, to report it, with the two after + it that keep that report for the run's Discussions post (#2030). """ for name, job_id, install in INSTALLS: @@ -647,7 +648,8 @@ def test_a_leg_that_cannot_install_stops_there(self) -> None: condition = str(step.get('if', '')) if 'matrix.coverage' in condition: continue # the `latest` leg only, which a failed install fails anyway - if step.get('name') in ('KTLO summary', 'Experimental summary'): + if step.get('name') in ('KTLO summary', 'Experimental summary', 'Name the job summary', + 'Upload the job summary'): continue with self.subTest(workflow=name, job=job_id, step=step.get('name') or step.get('uses')): self.assertIn("steps.install.outcome == 'success'", condition) diff --git a/tests/project/test_resource_monitor_1740.py b/tests/project/test_resource_monitor_1740.py index 74b8b20bc7..f92cf79eeb 100644 --- a/tests/project/test_resource_monitor_1740.py +++ b/tests/project/test_resource_monitor_1740.py @@ -545,7 +545,9 @@ def test_only_the_experimental_job_runs_it(self) -> None: with self.subTest(workflow=path.name): self.assertNotIn('resource-monitor', path.read_text(encoding='utf-8')) self.assertIs(job(*EXPERIMENTAL).get('continue-on-error'), True) - self.assertEqual(steps()[-1]['name'], 'Experimental summary') + # #2030: last but the two that name and upload the job's summary. + self.assertEqual([entry['name'] for entry in steps()[-3:]], + ['Experimental summary', 'Name the job summary', 'Upload the job summary']) def test_the_caps_and_check_name_are_unchanged(self) -> None: spec = job(*EXPERIMENTAL) diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 7bd03a736b..3dd90b45e7 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -64,6 +64,8 @@ INTEGRATION_STEP = 'Run full test suite' #: Every workflow that waits on "Unit Tests" completing, with ``workflow_run``. CONSUMERS = ('coverage-comment.yml', 'deploy-pages.yml', 'cron-vendor.yml', 'cron-conda.yml') +#: The workflows that wait on "Experimental Tests" completing, to post its report alone (#2030). +REPORTERS = ('ci-reports.yml',) #: Each job's ``timeout-minutes``, and its capped steps', by ``id``: ``test``'s are #: main's, ``experimental``'s are sized for GraalPy (88-113 minutes, estimate). #: #1938 adds the watcher's interpreter and its requests, capped too. @@ -481,7 +483,11 @@ def classify(spec: 'dict[str, Any]') -> 'str': self.assertIn("needs.changes.outputs.code != 'false'", self.unit['jobs']['test']['if']) def test_nothing_waits_on_it(self) -> None: - """No ``workflow_run`` names it, nothing calls it, and no ``needs:`` anywhere names its job.""" + """No ``workflow_run`` names it but a reporter's, nothing calls it, and no ``needs:`` anywhere names its job. + + #2030: :data:`REPORTERS` take its finished runs to Discussions, which holds up nothing. + + """ self.assertNotIn('workflow_call', triggers(self.own)) consumers = [] for path in sorted((ROOT / '.github' / 'workflows').glob('*.yml')): @@ -491,7 +497,7 @@ def test_nothing_waits_on_it(self) -> None: consumers.append(path.name) text = path.read_text(encoding='utf-8') with self.subTest(workflow=path.name): - self.assertNotIn(self.own['name'], waits) + self.assertIs(self.own['name'] in waits, path.name in REPORTERS) code = [line for line in text.splitlines() if not line.lstrip().startswith('#')] self.assertFalse([line for line in code if 'experimental.yml' in line]) for job_id, job_spec in (spec.get('jobs') or {}).items(): @@ -521,7 +527,9 @@ def test_a_failure_never_turns_the_run_red(self) -> None: spec = self.own['jobs']['experimental'] self.assertIs(spec.get('continue-on-error'), True) self.assertEqual(spec['env'], {'ANNOTATION_LEVEL': 'warning'}) - self.assertEqual(spec['steps'][-1]['name'], 'Experimental summary') + # #2030: the last step but the two that name and upload the job's summary. + self.assertEqual([entry['name'] for entry in spec['steps'][-3:]], + ['Experimental summary', 'Name the job summary', 'Upload the job summary']) expected = {'experimental.yml': ['experimental'], 'unit-tests.yml': ['test-ktlo', 'integration-ktlo', 'pypcap-parity-ktlo']} for name in ('unit-tests.yml', 'experimental.yml'): diff --git a/tests/project/test_unit_tests_push_reports_2030.py b/tests/project/test_unit_tests_push_reports_2030.py index c4a8475a50..c51a847d6e 100644 --- a/tests/project/test_unit_tests_push_reports_2030.py +++ b/tests/project/test_unit_tests_push_reports_2030.py @@ -1,59 +1,74 @@ # -*- coding: utf-8 -*- -"""#2030: Coverage Comment's ``discussion`` job, which posts a Unit Tests push run to Discussions. +"""#2030: the test workflows' runs posted to Discussions, through util/check_suite_report.py. ``unit-tests.yml`` is a reusable workflow whose callers grant ``contents: read``, so a Unit Tests run on a push reaches its thread through :file:`coverage-comment.yml`'s ``workflow_run`` relay, as a pull request's -coverage reaches its comment. +coverage reaches its comment; Experimental Tests and Python Compatibility reach +theirs through :file:`ci-reports.yml`'s. Both relays run the same steps. * :class:`TestTheJobs` pins the layout: ``comment`` keeps the pull-request path, - its ``if:`` and its scopes; ``discussion`` takes push runs alone and is the one - job holding ``discussions: write``; a push run's concurrency group is its own; - and the checkout is the default branch's script, never the run's head. -* :class:`TestTheReportStep` runs ``Write the run report`` with a stand-in - ``gh``: the conclusion each status maps from, ``warning`` for a green run - whose non-blocking legs warned or failed, the pinned list of annotations main's - KTLO legs write by design, which leave it ``success``, and the report it writes. + its ``if:`` and its scopes; each ``discussion`` takes the events it posts and is + the one job holding ``discussions: write``; a relayed run's concurrency group + is its own; the checkout is the default branch's scripts, never the run's head; + and the two relays' steps are one. +* :class:`TestTheScriptsWorkflows` holds the script's :data:`WORKFLOWS` to the + workflow files: their names, file stems, by-design lists and gated legs. +* :class:`TestTheJobSummaries` pins how each job keeps a copy of its summary and + uploads it, since the API serves none. +* :class:`TestTheReport` runs ``Write the run report`` with a stand-in ``gh``: + the status each run gets, the owner's push rule, pip's retries, a run that + tested nothing, and the job summaries folded in; :class:`TestTheDownload` and + :class:`TestTheFit` pin how the summaries are read and shared out, and + :class:`TestTheHelperCopies` how util/_actions.py copies a summary. * :class:`TestThePostStep` runs ``Post the report to Discussions`` with a stand-in ``python3``, and :class:`TestEndToEnd` runs both steps and the real - :file:`util/discussion_report.py` against the stand-in GitHub of + scripts against the stand-in GitHub of :mod:`tests.project.test_discussion_report_2030`. """ from __future__ import annotations -import ast import json import os import pathlib +import re import shutil import subprocess # nosec: B404 import sys import tempfile import textwrap import unittest +import unittest.mock from typing import TYPE_CHECKING from tests._workflows import WORKFLOWS, load, pinned from tests.project.test_discussion_report_2030 import (CATEGORY, MARKER, _FakeGitHub) -from tests.project.test_release_gates import job_blocks, job_runs +from tests.project.test_ktlo_legs_stay_green_1672 import interpolate +from tests.project.test_python_matrix_tiers_1633 import legs +from tests.project.test_release_gates import evaluate_condition, job_blocks, job_runs +from util import _actions, check_suite_report, discussion_report if TYPE_CHECKING: from typing import Any, Optional ROOT = pathlib.Path(__file__).resolve().parents[2] -SCRIPT = ROOT / 'util' / 'discussion_report.py' +SCRIPTS = ('_actions.py', 'check_suite_report.py', 'discussion_report.py') BASH = shutil.which('bash') NAME = 'coverage-comment.yml' +RELAY = 'ci-reports.yml' +#: The workflow each relay reports, by its file. +REPORTED = {NAME: ('unit-tests.yml',), RELAY: ('experimental.yml', 'python-compatibility.yml')} TITLE = 'unit-tests: main push runs' -REPORT = 'report/unit-tests-report.md' +REPORT = 'report/run-report.md' RUN_URL = 'https://github.com/JarryShaw/PyPCAPKit/actions/runs/39718608950' +RUN_ID = '39718608950' SHA = 'da8596414e3e6a53f43b20b85c86755cc2ed6686' SUITE = 'CS_kwDOBotKYc8AAAAZD_cS-g' -POSTER = {'checks': 'read', 'contents': 'read', 'discussions': 'write'} +POSTER = {'actions': 'read', 'checks': 'read', 'contents': 'read', 'discussions': 'write'} #: The pull-request comment's condition and scopes, as before #2030. COMMENT_IF = ("${{ github.event.workflow_run.event == 'pull_request' && " "github.event.workflow_run.conclusion != 'cancelled' }}") @@ -65,68 +80,103 @@ 'neutral': 'skipped', 'stale': 'cancelled', '': 'failure', } RUN = 'python3 util/discussion_report.py' +WRITE = 'python3 util/check_suite_report.py --report report/run-report.md' +#: The ``needs`` a job's name reads: ``gate``'s, the latest version. +NEEDS = {'python-versions': {'outputs': {'latest': '3.15'}}} +#: The step outputs ``Write the run report`` hands on. +OUTPUTS = ('post', 'title', 'mode', 'status') +#: The job summary's copy, and the two steps that end each job that writes one. +COPY = '${{ runner.temp }}/job-summary.md' +SAVE, UPLOAD = 'Name the job summary', 'Upload the job summary' +#: Each job that writes its summary, as ``(workflow, job)``: the KTLO summaries, +#: the coverage table, the engine cells' table and the experimental summaries. +SUMMARISED = ( + ('unit-tests.yml', 'test'), ('unit-tests.yml', 'test-ktlo'), ('unit-tests.yml', 'integration'), + ('unit-tests.yml', 'integration-ktlo'), ('unit-tests.yml', 'engine-tests'), + ('unit-tests.yml', 'pypcap-parity'), ('unit-tests.yml', 'pypcap-parity-ktlo'), + ('experimental.yml', 'experimental'), ('python-compatibility.yml', 'compatibility'), +) +#: The steps that copy the summary through util/_actions.py's JOB_SUMMARY_COPY. +COPYING = ('Report coverage', 'KTLO summary', 'Experimental summary') +#: The shell's copy of a summary line. +TEE = '| tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY"' -def discussion() -> 'dict[str, Any]': - return load(NAME)['jobs']['discussion'] +def discussion(name: 'str' = NAME) -> 'dict[str, Any]': + return load(name)['jobs']['discussion'] -def named(name: 'str') -> 'dict[str, Any]': - [step] = [entry for entry in discussion()['steps'] if entry.get('name') == name] +def named(name: 'str', workflow: 'str' = NAME) -> 'dict[str, Any]': + [step] = [entry for entry in discussion(workflow)['steps'] if entry.get('name') == name] return step +def note(level: 'str', title: 'str', message: 'str' = 'Process completed with exit code 1.') -> 'dict[str, str]': + return {'annotationLevel': level, 'title': title, 'message': message} + + def check_runs(*runs: 'tuple[str, str, list[tuple[str, ...]]]', more: 'bool' = False) -> 'dict[str, Any]': """The check-suite answer: ``(name, conclusion, [(level, title[, message])])`` per job.""" return {'data': {'node': {'checkRuns': {'pageInfo': {'hasNextPage': more}, 'nodes': [ - {'name': name, 'conclusion': conclusion, 'annotations': {'nodes': [ - {'annotationLevel': note[0], 'title': note[1], - 'message': note[2] if len(note) > 2 else 'Process completed with exit code 1.'} - for note in notes]}} + {'name': name, 'conclusion': conclusion, 'annotations': {'nodes': [note(*entry) for entry in notes]}} for name, conclusion, notes in runs]}}}} +def nodes(*runs: 'tuple[str, str, list[tuple[str, ...]]]') -> 'list[dict[str, Any]]': + return check_runs(*runs)['data']['node']['checkRuns']['nodes'] + + #: A job with the runner's notice alone. GREEN = ('Python 3.14', 'SUCCESS', [('NOTICE', 'The ubuntu-latest label will migrate')]) #: A KTLO leg whose tests failed, which ``KTLO summary`` reports as a warning (#1672). KTLO = ('Python 3.8 (KTLO)', 'SUCCESS', [('WARNING', 'tests failed on Python 3.8 (KTLO, non-blocking)'), ('FAILURE', '')]) - -#: The annotations main's run writes by design, as the step's ``EXPECTED`` lists them: -#: none since #2051 dropped the 3.7 legs and #2040's cap let pypy3.10 install. -BY_DESIGN = {} # type: dict[str, tuple[tuple[str, str], ...]] +#: A blocking job that passed with a warning, as a problem matcher writes one. +BLOCKING = ('Python 3.13', 'SUCCESS', [('WARNING', '', 'DeprecationWarning: something old')]) +#: pip retrying a request, as Unit Tests run 39846357077's ``Project tests`` showed it. +RETRY = ('WARNING', '', "WARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) " + "after connection broken by 'ReadTimeoutError(\"HTTPSConnectionPool(host='pypi.org', " + "port=443): Read timed out. (read timeout=15)\")': /simple/scapy/") +#: An engine's KTLO cell that failed, in a blocking job (#1672). +ENGINE_KTLO = ('Engines (Scapy)', 'SUCCESS', [('WARNING', 'Scapy failed on Python 3.9 (KTLO, non-blocking)'), + ('FAILURE', '')]) #: The list as it stood for pypy3.10's install failure, before #2040, which the -#: tests of the matching itself install in place of :data:`BY_DESIGN`. -LISTED = { - job: (('WARNING', f'install failed on {leg} (KTLO, non-blocking)'), ('FAILURE', ''), - ('WARNING', f'pcapkit did not install on {leg} (KTLO, non-blocking)')) - for job, leg in (('Python pypy3.10 (KTLO)', 'Python pypy3.10'),) -} -#: Unit Tests run 39718608950's jobs with any annotation but a notice, as the -#: check-suite query returned them on 2026-10-22, and one job with a notice alone. -#: Its 3.7 legs are left out: #2051 dropped them, and their entries in -#: ``EXPECTED`` with them. +#: tests of the matching itself pass in place of a workflow's empty one. +LISTED = {'Python pypy3.10 (KTLO)': (('WARNING', 'install failed on Python pypy3.10 (KTLO, non-blocking)'), + ('FAILURE', ''), + ('WARNING', 'pcapkit did not install on Python pypy3.10 (KTLO, non-blocking)'))} +#: Unit Tests run 39718608950's pypy3.10 job, as the check-suite query returned it on 2026-10-22. REAL_KTLO = "KTLO legs never block (#1633), so this job still succeeds (#1672); see the install step's log." -REAL_RUN = ( - ('Python pypy3.10 (KTLO)', 'SUCCESS', [ - ('WARNING', 'install failed on Python pypy3.10 (KTLO, non-blocking)', REAL_KTLO), - ('FAILURE', '', 'Process completed with exit code 1.'), - ('WARNING', 'pcapkit did not install on Python pypy3.10 (KTLO, non-blocking)', - "pip failed without an ERROR line; see this step's log. The rest of this leg is skipped."), - ('NOTICE', '', '"The ubuntu-latest label will migrate to Ubuntu 26 beginning October 19, 2026. For more ' - 'information, see https://github.com/actions/runner-images/issues/14748"')]), - GREEN, -) - -#: A stand-in ``gh`` for the check-suite query, logging its arguments; ``api -#: graphql --input -``, the script's own calls, goes to the stand-in GitHub. +PYPY = ('Python pypy3.10 (KTLO)', 'SUCCESS', [ + ('WARNING', 'install failed on Python pypy3.10 (KTLO, non-blocking)', REAL_KTLO), + ('FAILURE', '', 'Process completed with exit code 1.'), + ('WARNING', 'pcapkit did not install on Python pypy3.10 (KTLO, non-blocking)', + "pip failed without an ERROR line; see this step's log. The rest of this leg is skipped."), + ('NOTICE', '', 'The ubuntu-latest label will migrate to Ubuntu 26')]) + +#: A stand-in ``gh``: ``api graphql -f ...``, the check-suite query, answers +#: ``$FAKE_SUITE_ANSWER``; ``run download`` copies ``$FAKE_ARTIFACTS``' folders; +#: both log their arguments to ``$FAKE_SUITE_LOG``. ``api graphql --input -``, the +#: posting script's own calls, goes to the stand-in GitHub beside it. FAKE_GH = textwrap.dedent('''\ - import json, os, runpy, sys + import json, os, runpy, shutil, sys if '--input' in sys.argv: runpy.run_path(os.path.join(os.path.dirname(__file__), 'discussions.py'), run_name='__main__') sys.exit(0) with open(os.environ['FAKE_SUITE_LOG'], 'a', encoding='utf-8') as file: file.write(json.dumps(sys.argv[1:]) + '\\n') + if sys.argv[1:3] == ['run', 'download']: + source = os.environ.get('FAKE_ARTIFACTS') + if source == 'fail': + print('HTTP 403: Resource not accessible by integration', file=sys.stderr) + sys.exit(1) + if not source or not os.listdir(source): + print('no valid artifacts found to download', file=sys.stderr) + sys.exit(1) + target = sys.argv[sys.argv.index('--dir') + 1] + for name in os.listdir(source): + shutil.copytree(os.path.join(source, name), os.path.join(target, name)) + sys.exit(0) answer = os.environ.get('FAKE_SUITE_ANSWER') if not answer: print('gh: HTTP 403', file=sys.stderr) @@ -136,8 +186,33 @@ def check_runs(*runs: 'tuple[str, str, list[tuple[str, ...]]]', more: 'bool' = F ''') +def write_launcher(directory: 'pathlib.Path', name: 'str', script: 'str') -> None: + """``directory/name``, running ``script`` (beside it) on this interpreter.""" + (directory / f'{name}.py').write_text(script, encoding='utf-8') + launcher = directory / name + launcher.write_text(f'#!/bin/sh\nexec \'{sys.executable}\' "$(dirname "$0")/{name}.py" "$@"\n', + encoding='utf-8') + launcher.chmod(0o755) + + +def run_step(name: 'str', cwd: 'pathlib.Path', env: 'dict[str, str]', + workflow: 'str' = NAME) -> 'subprocess.CompletedProcess[str]': + """Step ``name``'s ``run:`` as a bash step runs it.""" + script = cwd.parent / 'step.sh' + script.write_text(named(name, workflow)['run'], encoding='utf-8') + return subprocess.run([str(BASH), '--noprofile', '--norc', '-eo', 'pipefail', str(script)], # nosec: B603 + cwd=cwd, env=env, capture_output=True, text=True, check=False, timeout=120) + + +def checkout(work: 'pathlib.Path') -> None: + """The scripts the job's sparse checkout holds, in ``work/util``.""" + (work / 'util').mkdir(parents=True) + for script in SCRIPTS: + shutil.copy(ROOT / 'util' / script, work / 'util') + + class TestTheJobs(unittest.TestCase): - """The relay's two jobs, and what each may do.""" + """The relays' jobs, and what each may do.""" def test_the_pull_request_comment_is_as_it_was(self) -> None: spec = load(NAME) @@ -149,18 +224,41 @@ def test_the_pull_request_comment_is_as_it_was(self) -> None: self.assertNotIn('discussion_report', json.dumps(comment)) def test_each_job_takes_its_own_events(self) -> None: - """``comment`` a pull request's run that was not cancelled; ``discussion`` every push run.""" + """``comment`` a pull request's run not cancelled; ``discussion`` a push, and in the relay a schedule too.""" jobs = job_blocks((WORKFLOWS / NAME).read_text(encoding='utf-8')) - for event in ('push', 'pull_request', 'merge_group', 'schedule'): + relay = job_blocks((WORKFLOWS / RELAY).read_text(encoding='utf-8')) + self.assertEqual(sorted(relay), ['discussion']) + for event in ('push', 'pull_request', 'merge_group', 'schedule', 'workflow_dispatch'): for conclusion in CONCLUSIONS: context = {'github': {'event': {'workflow_run': {'event': event, 'conclusion': conclusion}}}} with self.subTest(event=event, conclusion=conclusion): self.assertEqual(job_runs(jobs, 'discussion', context), event == 'push') self.assertEqual(job_runs(jobs, 'comment', context), event == 'pull_request' and conclusion != 'cancelled') + self.assertEqual(job_runs(relay, 'discussion', context), event in ('push', 'schedule')) + + def test_the_relay_waits_on_the_workflows_it_reports(self) -> None: + for relay, files in REPORTED.items(): + spec = load(relay) + with self.subTest(relay=relay): + self.assertEqual(spec['on'], {'workflow_run': { + 'workflows': [load(name)['name'] for name in files], 'types': ['completed']}}) + self.assertEqual(spec['permissions'], {}) + self.assertEqual(spec['env'], {'REPORT_CATEGORY': CATEGORY}) + + def test_the_reported_workflows_gain_no_write_scope(self) -> None: + """Posting is the relays'; ``unit-tests.yml`` stays callable with ``contents: read``.""" + for name in (name for files in REPORTED.values() for name in files): + spec = load(name) + with self.subTest(workflow=name): + self.assertEqual(spec['permissions'], {'contents': 'read'}) + for job_id, job in spec['jobs'].items(): + scopes = job.get('permissions') or {} + self.assertNotIn('write', json.dumps(scopes), job_id) def test_only_the_posting_job_may_write_discussions(self) -> None: - self.assertEqual(discussion()['permissions'], POSTER) + for relay in REPORTED: + self.assertEqual(discussion(relay)['permissions'], POSTER) for path in sorted(WORKFLOWS.glob('*.yml')): spec = load(path.name) with self.subTest(workflow=path.name): @@ -170,41 +268,65 @@ def test_only_the_posting_job_may_write_discussions(self) -> None: self.assertEqual(job_id, 'discussion') self.assertIn(RUN, json.dumps(job)) - def test_a_push_runs_group_is_its_own(self) -> None: - """A pull request's group is the one it was; a pending push relay is never cancelled by the next.""" + def test_a_relayed_runs_group_is_its_own(self) -> None: + """A pull request's group is the one it was; a pending relay is never cancelled by the next.""" group = ' '.join(load(NAME)['concurrency']['group'].split()) run = 'github.event.workflow_run' self.assertEqual(group, ( f"coverage-comment-${{{{ {run}.event == 'pull_request' && " f"format('{{0}}-{{1}}', {run}.head_repository.full_name, {run}.head_branch) || {run}.id }}}}")) + self.assertEqual(load(RELAY)['concurrency'], {'group': f'ci-reports-${{{{ {run}.id }}}}', + 'cancel-in-progress': False}) self.assertIs(load(NAME)['concurrency']['cancel-in-progress'], False) - def test_it_posts_with_the_script_in_push_mode(self) -> None: + def test_the_relays_run_one_set_of_steps(self) -> None: + mine = {key: value for key, value in discussion(NAME).items() if key not in ('if', 'name')} + theirs = {key: value for key, value in discussion(RELAY).items() if key not in ('if', 'name')} + self.assertEqual(mine, theirs) + + def test_it_writes_with_one_script_and_posts_with_the_other(self) -> None: job = discussion() - self.assertEqual(load(NAME)['env'], {'REPORT_CATEGORY': CATEGORY}) self.assertIs(job['continue-on-error'], True) for name in ('Write the run report', 'Post the report to Discussions'): with self.subTest(step=name): self.assertIs(named(name)['continue-on-error'], True) + write = named('Write the run report') + self.assertEqual(write['id'], 'report') + self.assertEqual(write['run'], WRITE) + run = 'github.event.workflow_run' + self.assertEqual(write['env'], { + 'GH_TOKEN': '${{ github.token }}', 'WORKFLOW': f'${{{{ {run}.name }}}}', + 'SUITE': f'${{{{ {run}.check_suite_node_id }}}}', 'RUN_ID': f'${{{{ {run}.id }}}}', + 'RUN_URL': f'${{{{ {run}.html_url }}}}', 'RUN_EVENT': f'${{{{ {run}.event }}}}', + 'RUN_CONCLUSION': f'${{{{ {run}.conclusion }}}}', 'HEAD_SHA': f'${{{{ {run}.head_sha }}}}', + 'HEAD_BRANCH': f'${{{{ {run}.head_branch }}}}'}) post = named('Post the report to Discussions') + self.assertEqual(post['if'], "${{ steps.report.outputs.post == 'true' }}") command = ' '.join(post['run'].replace('\\\n', ' ').split()) self.assertEqual(command, ( - f"GITHUB_WORKFLOW='Unit Tests' {RUN} --category \"$REPORT_CATEGORY\" --title \"$TITLE\" " - f'--report {REPORT} --mode push --status "$STATUS" --run-url "$RUN_URL" --sha "$HEAD_SHA"')) - run = 'github.event.workflow_run' + f'GITHUB_WORKFLOW="$WORKFLOW" {RUN} --category "$REPORT_CATEGORY" --title "$TITLE" ' + f'--report {REPORT} --mode "$MODE" --status "$STATUS" --run-url "$RUN_URL" --sha "$HEAD_SHA"')) self.assertEqual(post['env'], { - 'GH_TOKEN': '${{ github.token }}', 'TITLE': '${{ steps.report.outputs.title }}', + 'GH_TOKEN': '${{ github.token }}', 'WORKFLOW': f'${{{{ {run}.name }}}}', + 'TITLE': '${{ steps.report.outputs.title }}', 'MODE': '${{ steps.report.outputs.mode }}', 'STATUS': '${{ steps.report.outputs.status }}', 'RUN_URL': f'${{{{ {run}.html_url }}}}', 'HEAD_SHA': f'${{{{ {run}.head_sha }}}}'}) - self.assertEqual(named('Write the run report')['id'], 'report') + + def test_the_post_step_runs_only_on_a_report_to_post(self) -> None: + condition = named('Post the report to Discussions')['if'] + for value, runs in (('true', True), ('false', False), ('', False)): + context = {'steps': {'report': {'outputs': {'post': value}}}} + with self.subTest(post=value): + self.assertIs(evaluate_condition(condition, context), runs) def test_it_never_checks_out_the_runs_head(self) -> None: - """The default branch's script alone, as ``workflow_run`` checks out; the head is data in ``env:``.""" + """The default branch's scripts alone, as ``workflow_run`` checks out; the head is data in ``env:``.""" uses = [entry for entry in discussion()['steps'] if 'uses' in entry] self.assertEqual(len(uses), 1) self.assertTrue(pinned(uses[0]['uses'], 'actions/checkout'), uses[0]['uses']) - self.assertEqual(uses[0]['with'], {'sparse-checkout': 'util/discussion_report.py\n', - 'sparse-checkout-cone-mode': False, 'persist-credentials': False}) + self.assertEqual(uses[0]['with'], { + 'sparse-checkout': ''.join(f'util/{script}\n' for script in SCRIPTS), + 'sparse-checkout-cone-mode': False, 'persist-credentials': False}) for entry in discussion()['steps']: outside = {key: value for key, value in entry.items() if key != 'env'} with self.subTest(step=entry.get('name') or entry['uses']): @@ -212,31 +334,117 @@ def test_it_never_checks_out_the_runs_head(self) -> None: self.assertNotIn('${{', entry.get('run', '')) -def write_launcher(directory: 'pathlib.Path', name: 'str', script: 'str') -> None: - """``directory/name``, running ``script`` (beside it) on this interpreter.""" - (directory / f'{name}.py').write_text(script, encoding='utf-8') - launcher = directory / name - launcher.write_text(f'#!/bin/sh\nexec \'{sys.executable}\' "$(dirname "$0")/{name}.py" "$@"\n', - encoding='utf-8') - launcher.chmod(0o755) - - -def run_step(name: 'str', cwd: 'pathlib.Path', env: 'dict[str, str]', - expected: 'Optional[dict[str, Any]]' = None) -> 'subprocess.CompletedProcess[str]': - """Step ``name``'s ``run:`` as a bash step runs it, with ``EXPECTED`` replaced by ``expected`` if given.""" - script = cwd.parent / 'step.sh' - source = named(name)['run'] - if expected is not None: - assert source.count('\nEXPECTED = {}\n') == 1, 'the step no longer reads EXPECTED = {}' - source = source.replace('\nEXPECTED = {}\n', f'\nEXPECTED = {expected!r}\n') - script.write_text(source, encoding='utf-8') - return subprocess.run([str(BASH), '--noprofile', '--norc', '-eo', 'pipefail', str(script)], # nosec: B603 - cwd=cwd, env=env, capture_output=True, text=True, check=False, timeout=120) +class TestTheScriptsWorkflows(unittest.TestCase): + """:data:`check_suite_report.WORKFLOWS` against the workflow files.""" + + def test_it_reports_the_relayed_workflows(self) -> None: + files = [name for names in REPORTED.values() for name in names] + self.assertEqual(sorted(check_suite_report.WORKFLOWS), sorted(load(name)['name'] for name in files)) + for name in files: + with self.subTest(workflow=name): + self.assertEqual(check_suite_report.WORKFLOWS[load(name)['name']].stem, name[:-len('.yml')]) + + def test_no_annotation_is_by_design(self) -> None: + """The owner's rulings: each list is empty (#2051, #2040 for Unit Tests; #2030 for the others).""" + for name, workflow in check_suite_report.WORKFLOWS.items(): + with self.subTest(workflow=name): + self.assertEqual(workflow.expected, {}) + + def test_the_legs_are_the_gated_jobs(self) -> None: + """``legs`` names each job ``current`` or ``changes.code`` can skip, and no other; Compat's matrix alone.""" + gates = ("needs.current.outputs.current != 'false'", "needs.changes.outputs.code != 'false'") + for name in (name for names in REPORTED.values() for name in names): + pattern = check_suite_report.WORKFLOWS[load(name)['name']].legs + for job_id, job in load(name)['jobs'].items(): + gated = any(gate in str(job.get('if', '')) for gate in gates) or job_id == 'compatibility' + gated = gated and job_id not in ('python-versions', 'changes') + for leg in (legs(name, job_id) if 'strategy' in job else [{}]): + title = interpolate(str(job.get('name', job_id)), {'matrix': leg, 'needs': NEEDS}) + with self.subTest(workflow=name, job=job_id, name=title): + self.assertIs(bool(pattern.match(title)), gated) + + +class TestTheJobSummaries(unittest.TestCase): + """Each job that writes its summary keeps a copy, names it, and uploads both (#2030).""" + + def test_each_summary_step_copies_its_lines(self) -> None: + found = set() + for name, job_id in SUMMARISED: + for step in load(name)['jobs'][job_id]['steps']: + text = step.get('run', '') + with self.subTest(workflow=name, job=job_id, step=step.get('name')): + if step.get('name') in COPYING: + found.add(step['name']) + self.assertEqual(step['env']['JOB_SUMMARY_COPY'], COPY) + else: + self.assertNotIn('JOB_SUMMARY_COPY', step.get('env') or {}) + self.assertEqual(text.count('"$GITHUB_STEP_SUMMARY"'), text.count(TEE)) + self.assertEqual(found, set(COPYING)) + + def test_no_test_step_carries_the_copy(self) -> None: + """A test writing the summary through util/_actions.py would write the job's copy.""" + for path in sorted(WORKFLOWS.glob('*.yml')): + spec = load(path.name) + self.assertNotIn('JOB_SUMMARY_COPY', json.dumps(spec.get('env') or {})) + for job_id, job in spec['jobs'].items(): + with self.subTest(workflow=path.name, job=job_id): + self.assertNotIn('JOB_SUMMARY_COPY', json.dumps(job.get('env') or {})) + for step in job.get('steps', []): + if 'JOB_SUMMARY_COPY' in (step.get('env') or {}): + self.assertIn(step['name'], COPYING) + self.assertNotIn('pytest', step['run']) + + def test_every_writer_is_listed(self) -> None: + """No other job of the reported workflows writes a summary, so none is left out.""" + writers = re.compile(r'GITHUB_STEP_SUMMARY|util/(?:leg_summary|coverage_report)\.py') + for name in (name for names in REPORTED.values() for name in names): + for job_id, job in load(name)['jobs'].items(): + if (name, job_id) in SUMMARISED or job_id == 'current': + continue + with self.subTest(workflow=name, job=job_id): + self.assertIsNone(writers.search(json.dumps(job.get('steps', [])))) + + def test_each_job_names_and_uploads_it(self) -> None: + for name, job_id in SUMMARISED: + job = load(name)['jobs'][job_id] + save, upload = job['steps'][-2:] + with self.subTest(workflow=name, job=job_id): + self.assertEqual((save['name'], upload['name']), (SAVE, UPLOAD)) + self.assertEqual(save['run'], 'if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf \'%s\\n\' ' + '"$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi') + self.assertTrue(pinned(upload['uses'], 'actions/upload-artifact'), upload['uses']) + self.assertEqual(upload['with'], { + 'name': 'job-summary-${{ github.job }}-${{ strategy.job-index }}', + 'path': f'{COPY}\n${{{{ runner.temp }}}}/job-summary.name\n', + 'if-no-files-found': 'ignore', 'retention-days': 7}) + for step in (save, upload): + self.assertEqual(step['if'], '${{ always() }}') + self.assertIs(step['continue-on-error'], True) + # The name the job's check run carries, on every leg. + for leg in legs(name, job_id): + self.assertEqual(interpolate(save['env']['JOB_NAME'], {'matrix': leg}), + interpolate(job['name'], {'matrix': leg})) + + @unittest.skipIf(not BASH, 'needs bash') + def test_the_name_is_written_beside_a_summary_alone(self) -> None: + save = load('unit-tests.yml')['jobs']['test']['steps'][-2] + with tempfile.TemporaryDirectory() as tmp: + temp = pathlib.Path(tmp) + for text in ('', '| table |\n'): + (temp / 'job-summary.md').write_text(text, encoding='utf-8') + result = subprocess.run([str(BASH), '--noprofile', '--norc', '-eo', 'pipefail', '-c', save['run']], + env={'RUNNER_TEMP': tmp, 'JOB_NAME': 'Python 3.8 (KTLO)', + 'PATH': os.environ.get('PATH', '')}, + capture_output=True, text=True, check=False, timeout=60) + self.assertEqual(result.returncode, 0, result.stderr) + with self.subTest(summary=bool(text)): + self.assertEqual((temp / 'job-summary.name').exists(), bool(text)) + self.assertEqual((temp / 'job-summary.name').read_text(encoding='utf-8'), 'Python 3.8 (KTLO)\n') @unittest.skipIf(not BASH or os.name == 'nt', 'needs bash and a POSIX shell') -class TestTheReportStep(unittest.TestCase): - """``Write the run report``, against a stand-in ``gh`` answering the check-suite query.""" +class TestTheReport(unittest.TestCase): + """``Write the run report``, against a stand-in ``gh`` answering the check suite and the artifacts.""" def setUp(self) -> None: tmp = tempfile.TemporaryDirectory(prefix='pcapkit-2030-push-') # pylint: disable=consider-using-with @@ -247,42 +455,57 @@ def setUp(self) -> None: write_launcher(self.bin, 'gh', FAKE_GH) (self.bin / 'python3').symlink_to(sys.executable) self.work = self.root / 'work' - self.work.mkdir() + checkout(self.work) self.answer = self.root / 'answer.json' self.log = self.root / 'suite.log' self.output = self.root / 'output' - - def report(self, conclusion: 'str' = 'success', answer: 'Optional[object]' = None, - branch: 'str' = 'main', expected: 'Optional[dict[str, Any]]' = None, + self.artifacts = self.root / 'artifacts' + self.artifacts.mkdir() + + def artifact(self, folder: 'str', summary: 'str', job: 'Optional[str]' = None) -> None: + """An artifact of the run: a job's copy of its summary, and its name if given.""" + path = self.artifacts / folder + path.mkdir() + (path / 'job-summary.md').write_text(summary, encoding='utf-8') + if job is not None: + (path / 'job-summary.name').write_text(job + '\n', encoding='utf-8') + + def report(self, conclusion: 'str' = 'success', answer: 'Optional[object]' = None, branch: 'str' = 'main', + workflow: 'str' = 'Unit Tests', event: 'str' = 'push', **extra: 'str') -> 'tuple[dict[str, str], str, str]': - """The step's outputs, its report and its stdout, for a run that ended ``conclusion``.""" + """The step's outputs, its report (``''`` where it wrote none) and its stdout.""" shutil.rmtree(self.work / 'report', ignore_errors=True) self.output.write_text('', encoding='utf-8') env = {'PATH': f'{self.bin}{os.pathsep}{os.environ.get("PATH", "")}', 'FAKE_SUITE_LOG': str(self.log), 'GITHUB_OUTPUT': str(self.output), 'RUNNER_TEMP': str(self.root), 'GH_TOKEN': 'token', - 'SUITE': SUITE, 'RUN_URL': RUN_URL, 'RUN_CONCLUSION': conclusion, 'HEAD_SHA': SHA, - 'HEAD_BRANCH': branch, **extra} + 'GITHUB_REPOSITORY': 'JarryShaw/PyPCAPKit', 'FAKE_ARTIFACTS': str(self.artifacts), + 'WORKFLOW': workflow, 'SUITE': SUITE, 'RUN_ID': RUN_ID, 'RUN_URL': RUN_URL, 'RUN_EVENT': event, + 'RUN_CONCLUSION': conclusion, 'HEAD_SHA': SHA, 'HEAD_BRANCH': branch, **extra} if answer is not None: self.answer.write_text(answer if isinstance(answer, str) else json.dumps(answer), encoding='utf-8') env['FAKE_SUITE_ANSWER'] = str(self.answer) - result = run_step('Write the run report', self.work, env, expected) + result = run_step('Write the run report', self.work, env) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) outputs = dict(line.split('=', 1) for line in self.output.read_text(encoding='utf-8').splitlines()) - return outputs, (self.work / REPORT).read_text(encoding='utf-8'), result.stdout + self.assertEqual(sorted(outputs), sorted(OUTPUTS)) + path = self.work / REPORT + return outputs, path.read_text(encoding='utf-8') if path.exists() else '', result.stdout def test_each_conclusion_maps_to_a_status(self) -> None: for conclusion, status in CONCLUSIONS.items(): with self.subTest(conclusion=conclusion): outputs, report, _ = self.report(conclusion, check_runs(GREEN)) - self.assertEqual(outputs, {'title': TITLE, 'status': status}) + self.assertEqual(outputs, {'post': 'true', 'title': TITLE, 'mode': 'push', 'status': status}) self.assertIn(f'concluded `{conclusion or "unknown"}`.', report) - def test_it_asks_for_the_runs_check_suite(self) -> None: + def test_it_asks_for_the_runs_check_suite_and_summaries(self) -> None: self.report('success', check_runs(GREEN)) - [args] = [json.loads(line) for line in self.log.read_text(encoding='utf-8').splitlines()] - self.assertEqual(args[:2], ['api', 'graphql']) - self.assertEqual(args[-2:], ['-f', f'id={SUITE}']) - self.assertIn('checkRuns(first: 100)', args[3]) + suite, download = [json.loads(line) for line in self.log.read_text(encoding='utf-8').splitlines()] + self.assertEqual(suite[:2], ['api', 'graphql']) + self.assertEqual(suite[-2:], ['-f', f'id={SUITE}']) + self.assertIn('checkRuns(first: 100)', suite[3]) + self.assertEqual(download[:-1], ['run', 'download', RUN_ID, '--repo', 'JarryShaw/PyPCAPKit', + '--pattern', 'job-summary-*', '--dir']) def test_a_clean_green_run(self) -> None: outputs, report, _ = self.report('success', check_runs(GREEN)) @@ -302,63 +525,88 @@ def test_a_green_run_whose_ktlo_leg_warned_is_a_warning(self) -> None: '- `Python 3.8 (KTLO)`: `tests failed on Python 3.8 (KTLO, non-blocking)`; ' '`Process completed with exit code 1.`\n')) - def test_the_by_design_list_is_pinned(self) -> None: - """``EXPECTED`` holds the install failures main's KTLO legs write on every run, and nothing else.""" - source = named('Write the run report')['run'] - script = source.split("<<'PY'\n", 1)[1].split('\nPY\n', 1)[0] - [value] = [node.value for node in ast.parse(script).body if isinstance(node, ast.Assign) - and [getattr(target, 'id', None) for target in node.targets] == ['EXPECTED']] - self.assertEqual(ast.literal_eval(value), BY_DESIGN) - - def test_with_nothing_listed_the_old_install_failures_warn(self) -> None: - """Run 39718608950's pypy3.10 install failure is no longer by design (#2040).""" - outputs, report, _ = self.report('success', check_runs(*REAL_RUN)) - self.assertEqual(outputs['status'], 'warning') - self.assertIn('- `Python pypy3.10 (KTLO)`: `install failed on Python pypy3.10 (KTLO, non-blocking)`', report) - - def test_mains_by_design_annotations_leave_a_green_run_green(self) -> None: - """Run 39718608950's annotations, as the API returned them: ``success``, and none listed.""" - outputs, report, _ = self.report('success', check_runs(*REAL_RUN), expected=LISTED) - self.assertEqual(outputs['status'], 'success') - self.assertEqual(report, ( - '### Unit Tests push run\n\n' - f'[Unit Tests run]({RUN_URL}) for `{SHA[:9]}` concluded `success`.\n\n' - 'Every job passed, with no warning but the by-design ones.\n')) - - def test_a_reworded_message_still_matches(self) -> None: - """The title is matched, not the message, which #2041 rewords.""" - reworded = [(level, title, 'something else entirely') for level, title, _ in REAL_RUN[0][2]] - outputs, _, _ = self.report('success', check_runs(('Python pypy3.10 (KTLO)', 'SUCCESS', reworded)), - expected=LISTED) + def test_a_push_runs_passing_blocking_job_earns_no_post(self) -> None: + """The owner's push rule: its warnings neither count nor are listed.""" + outputs, report, _ = self.report('success', check_runs(GREEN, BLOCKING)) self.assertEqual(outputs['status'], 'success') + self.assertNotIn('DeprecationWarning', report) + self.assertIn('Every job passed, with no warning that counts.\n', report) + + def test_a_non_blocking_legs_warning_still_counts_on_a_push(self) -> None: + for job in (KTLO, ENGINE_KTLO, ('Python 3.15t (experimental)', 'SUCCESS', + [('WARNING', 'tests failed on Python 3.15t (experimental, non-blocking)')])): + with self.subTest(job=job[0]): + outputs, report, _ = self.report('success', check_runs(GREEN, BLOCKING, job)) + self.assertEqual(outputs['status'], 'warning') + self.assertIn(f'- `{job[0]}`: `{job[2][0][1]}`', report) + self.assertNotIn('DeprecationWarning', report) - def test_anything_else_still_warns(self) -> None: - """An annotation the list does not hold, beside the by-design ones or in their place, is a warning.""" - failed = 'FAILED tests/project/test_x.py::TestX::test_y - AssertionError' + def test_a_failed_push_run_lists_every_warning(self) -> None: + """Annotations on a passing blocking job matter once the run failed.""" + outputs, report, _ = self.report('failure', check_runs(GREEN, BLOCKING, ('Python 3.12', 'FAILURE', []))) + self.assertEqual(outputs['status'], 'failure') + self.assertIn('**Did not pass:**\n\n- `Python 3.12` (failure)\n', report) + self.assertIn('- `Python 3.13`: `DeprecationWarning: something old`', report) + + def test_a_scheduled_run_counts_every_warning(self) -> None: + """A run that posts every time keeps the rule as it was: any annotation on a passing job warns.""" + outputs, report, _ = self.report('success', check_runs(('Compat Python 3.13', 'SUCCESS', BLOCKING[2])), + workflow='Python Compatibility', event='schedule') + self.assertEqual(outputs, {'post': 'true', 'title': 'python-compatibility: main scheduled runs', + 'mode': 'scheduled', 'status': 'warning'}) + self.assertTrue(report.startswith('### Python Compatibility scheduled run\n\n' + f'[Python Compatibility run]({RUN_URL})'), report) + self.assertIn('- `Compat Python 3.13`: `DeprecationWarning: something old`', report) + + def test_pips_retries_never_count(self) -> None: + """On a non-blocking leg on a push, or any job on the schedule, where every other warning counts.""" + for workflow, event, job in (('Unit Tests', 'push', 'Python 3.8 (KTLO)'), + ('Python Compatibility', 'schedule', 'Compat Python 3.13'), + ('Python Compatibility', 'push', 'Compat Python 3.9 (KTLO)')): + with self.subTest(workflow=workflow, event=event, job=job): + outputs, report, _ = self.report('success', check_runs((job, 'SUCCESS', [RETRY])), + workflow=workflow, event=event) + self.assertEqual(outputs['status'], 'success') + self.assertNotIn('Retrying', report) + self.assertIn('Every job passed, with no warning that counts.\n', report) + + def test_a_retry_title_or_message_matches(self) -> None: + for annotation in (note(*RETRY), note('FAILURE', RETRY[2], 'x')): + with self.subTest(annotation=annotation): + self.assertTrue(check_suite_report.ignored('Project tests', annotation, {})) + self.assertFalse(check_suite_report.ignored('Project tests', note('WARNING', '', 'Retrying soon'), {})) + + def test_the_by_design_matching(self) -> None: + """A listed title, at its level, on its job; an empty prefix only an empty title; anything else warns.""" + found = check_suite_report.assess(nodes(PYPY, GREEN), 'success', 'scheduled', LISTED) + self.assertEqual(found, check_suite_report.Assessment('success', [], [], True)) + reworded = [(level, title, 'something else entirely') for level, title, _ in PYPY[2]] + found = check_suite_report.assess(nodes((PYPY[0], 'SUCCESS', reworded)), 'success', 'scheduled', LISTED) + self.assertEqual(found.status, 'success') cases = { - 'a failed test on a by-design job': ('Python pypy3.10 (KTLO)', 'WARNING', failed), 'a failed tests step on a by-design job': ( - 'Python pypy3.10 (KTLO)', 'WARNING', 'tests failed on Python pypy3.10 (KTLO, non-blocking)'), - 'a titled failure on a by-design job': ('Python pypy3.10 (KTLO)', 'FAILURE', 'Something broke'), + PYPY[0], 'WARNING', 'tests failed on Python pypy3.10 (KTLO, non-blocking)'), + 'a titled failure on a by-design job': (PYPY[0], 'FAILURE', 'Something broke'), 'a by-design title at another level': ( - 'Python pypy3.10 (KTLO)', 'FAILURE', 'install failed on Python pypy3.10 (KTLO, non-blocking)'), + PYPY[0], 'FAILURE', 'install failed on Python pypy3.10 (KTLO, non-blocking)'), 'a by-design title on another job': ( 'Python 3.8 (KTLO)', 'WARNING', 'install failed on Python pypy3.10 (KTLO, non-blocking)'), - 'an install failure on a leg not listed': ( - 'Python 3.8 (KTLO)', 'WARNING', 'install failed on Python 3.8 (KTLO, non-blocking)'), } for case, (job, level, title) in cases.items(): - runs = [(name, result, list(notes)) for name, result, notes in REAL_RUN] - found = [entry for entry in runs if entry[0] == job] - if found: - found[0][2].append((level, title, 'extra')) - else: - runs.append((job, 'SUCCESS', [(level, title, 'extra')])) - with self.subTest(case=case): - outputs, report, _ = self.report('success', check_runs(*runs), expected=LISTED) - self.assertEqual(outputs['status'], 'warning') - [line] = [line for line in report.splitlines() if line.startswith('- ')] - self.assertEqual(line, f'- `{job}`: `{title}`') + runs = [PYPY, GREEN] if job == PYPY[0] else [PYPY, GREEN, (job, 'SUCCESS', [])] + runs = [(name, result, list(notes) + ([(level, title, 'extra')] if name == job else [])) + for name, result, notes in runs] + for mode in ('push', 'scheduled'): + with self.subTest(case=case, mode=mode): + found = check_suite_report.assess(nodes(*runs), 'success', mode, LISTED) + self.assertEqual(found.status, 'warning') + self.assertEqual(found.warned, [f'`{job}`: `{title}`']) + + def test_with_nothing_listed_the_old_install_failures_warn(self) -> None: + """Run 39718608950's pypy3.10 install failure is no longer by design (#2040).""" + outputs, report, _ = self.report('success', check_runs(PYPY, GREEN)) + self.assertEqual(outputs['status'], 'warning') + self.assertIn('- `Python pypy3.10 (KTLO)`: `install failed on Python pypy3.10 (KTLO, non-blocking)`', report) def test_a_job_that_failed_without_failing_the_run_is_a_warning(self) -> None: outputs, report, _ = self.report('success', check_runs(GREEN, ('Python 3.9 (KTLO)', 'FAILURE', []))) @@ -366,25 +614,55 @@ def test_a_job_that_failed_without_failing_the_run_is_a_warning(self) -> None: self.assertIn('**Failed without failing the run:**\n\n- `Python 3.9 (KTLO)` (failure)\n', report) def test_a_failed_run_names_its_jobs(self) -> None: - answer = check_runs(*REAL_RUN, KTLO, ('Project tests', 'FAILURE', [('FAILURE', '')]), + answer = check_runs(PYPY, KTLO, ('Project tests', 'FAILURE', [('FAILURE', '')]), ('Required checks passed', 'FAILURE', []), ('Python 3.13', 'TIMED_OUT', [])) - outputs, report, _ = self.report('failure', answer, expected=LISTED) + outputs, report, _ = self.report('failure', answer) self.assertEqual(outputs['status'], 'failure') self.assertIn('**Did not pass:**\n\n- `Project tests` (failure)\n- `Required checks passed` (failure)\n' '- `Python 3.13` (timed_out)\n', report) self.assertIn('- `Python 3.8 (KTLO)`: ', report) - self.assertNotIn('pypy3.10', report) + + def test_a_run_that_tested_nothing_posts_nothing(self) -> None: + """Every leg skipped, as for a superseded commit (``current``) or no code change (``changes``).""" + superseded = check_runs(('Check the commit is still current', 'SUCCESS', []), + ('Read the Python versions', 'SKIPPED', []), ('Classify changed files', 'SKIPPED', []), + ('Python 3.15t (experimental)', 'SKIPPED', [])) + docs_only = check_runs(('Classify changed files', 'SUCCESS', []), ('Python 3.14', 'SKIPPED', []), + ('Engines (Scapy)', 'SKIPPED', []), ('Project tests', 'SUCCESS', []), + ('Required checks passed', 'SUCCESS', [])) + for workflow, answer in (('Experimental Tests', superseded), ('Unit Tests', docs_only)): + for conclusion in ('success', 'skipped'): + with self.subTest(workflow=workflow, conclusion=conclusion): + outputs, report, stdout = self.report(conclusion, answer, workflow=workflow) + self.assertEqual(outputs['post'], 'false') + self.assertEqual(report, '') + self.assertIn('nothing is posted', stdout) + + def test_a_run_that_failed_before_its_legs_still_posts(self) -> None: + answer = check_runs(('Check the commit is still current', 'SUCCESS', []), + ('Classify changed files', 'FAILURE', []), ('Python 3.15t (experimental)', 'SKIPPED', [])) + outputs, report, _ = self.report('failure', answer, workflow='Experimental Tests') + self.assertEqual((outputs['post'], outputs['status'], outputs['title']), + ('true', 'failure', 'experimental: main push runs')) + self.assertIn('- `Classify changed files` (failure)', report) + + def test_an_unknown_workflow_or_a_pull_request_posts_nothing(self) -> None: + outputs, _, stdout = self.report('success', check_runs(GREEN), workflow='Lint') + self.assertEqual(outputs['post'], 'false') + self.assertIn("::warning title=CI report not written::'Lint' is no workflow", stdout) + outputs, _, stdout = self.report('success', check_runs(GREEN), event='pull_request') + self.assertEqual(outputs['post'], 'false') + self.assertNotIn('::warning', stdout) def test_unreadable_jobs_keep_the_conclusion(self) -> None: for answer in (None, '', json.dumps({'data': {'node': None}})): for conclusion in ('success', 'failure'): with self.subTest(answer=answer, conclusion=conclusion): outputs, report, stdout = self.report(conclusion, answer) - self.assertEqual(outputs['status'], conclusion) + self.assertEqual((outputs['post'], outputs['status']), ('true', conclusion)) self.assertTrue(report.endswith("The run's jobs could not be read, so none is named here.\n"), report) - if answer is None: - self.assertIn("::warning::The run's jobs could not be read", stdout) + self.assertIn("::warning title=CI report not written::The run's jobs could not be read", stdout) def test_a_long_suite_says_it_was_cut(self) -> None: _, report, _ = self.report('success', check_runs(GREEN, more=True)) @@ -392,20 +670,122 @@ def test_a_long_suite_says_it_was_cut(self) -> None: def test_the_runs_text_renders_as_code(self) -> None: """A job name or annotation holding Markdown, a mention or a newline stays one inert code span.""" - noisy = ('Py `3` @JarryShaw', 'SUCCESS', [('WARNING', 'a\n**b** `c` @d ' + 'x' * 300)]) + noisy = ('Py `3` @JarryShaw (KTLO)', 'SUCCESS', [('WARNING', 'a\n**b** `c` @d ' + 'x' * 300)]) _, report, _ = self.report('success', check_runs(noisy), RUN_URL='javascript:alert(1)', HEAD_SHA='nope') [line] = [line for line in report.splitlines() if line.startswith('- ')] flat = "a **b** 'c' @d " - self.assertEqual(line, f"- `Py '3' @JarryShaw`: `{flat}{'x' * (197 - len(flat))}...`") + self.assertEqual(line, f"- `Py '3' @JarryShaw (KTLO)`: `{flat}{'x' * (197 - len(flat))}...`") self.assertIn('\nThe Unit Tests run concluded `success`.\n', report) - def test_the_branch_names_the_thread(self) -> None: - for branch, title in (('main', TITLE), ('release/1.4', 'unit-tests: release/1.4 push runs'), - ('a b$(x)`y`', 'unit-tests: a-b--x--y- push runs'), - ('', 'unit-tests: unknown push runs')): - with self.subTest(branch=branch): - outputs, _, _ = self.report('success', check_runs(GREEN), branch=branch) - self.assertEqual(outputs['title'], title) + def test_the_branch_and_mode_name_the_thread(self) -> None: + for branch, event, title in (('main', 'push', TITLE), + ('release/1.4', 'push', 'unit-tests: release/1.4 push runs'), + ('a b$(x)`y`', 'push', 'unit-tests: a-b--x--y- push runs'), + ('', 'push', 'unit-tests: unknown push runs'), + ('main', 'schedule', 'unit-tests: main scheduled runs'), + ('main', 'workflow_dispatch', 'unit-tests: main scheduled runs')): + with self.subTest(branch=branch, event=event): + outputs, _, _ = self.report('success', check_runs(GREEN), branch=branch, event=event) + self.assertEqual((outputs['title'], outputs['mode']), + (title, 'push' if event == 'push' else 'scheduled')) + + def test_the_job_summaries_are_folded_in(self) -> None: + """One ``
`` a job, in the check suite's order, named from the artifact; mentions neutralised.""" + self.artifact('job-summary-test-0', '| Package | Cover |\n| --- | --- |\n| `pcapkit` | 91% |\n', 'Python 3.14') + self.artifact('job-summary-test-ktlo-0', + '- Python 3.8 (KTLO, non-blocking): step `tests` failed; cc @someone\n', 'Python 3.8 (KTLO)') + self.artifact('job-summary-x-1', 'An unnamed summary.\n') + _, report, _ = self.report('success', check_runs(KTLO, GREEN)) + head, _, rest = report.partition('\n\n**Job summaries:**\n\n') + self.assertTrue(head.endswith('`Process completed with exit code 1.`'), head) + self.assertEqual(rest, ( + '
Python 3.8 (KTLO)\n\n' + '- Python 3.8 (KTLO, non-blocking): step `tests` failed; cc @\u2060someone\n\n
\n\n' + '
Python 3.14\n\n' + '| Package | Cover |\n| --- | --- |\n| `pcapkit` | 91% |\n\n
\n\n' + '
job-summary-x-1\n\nAn unnamed summary.\n\n
\n')) + + def test_a_summary_too_long_is_trimmed_with_a_link(self) -> None: + self.artifact('job-summary-test-0', 'short\n', 'Python 3.14') + self.artifact('job-summary-engine-tests-0', ''.join(f'| 3.{n % 10} | Scapy | supported | passed |\n' + for n in range(4000)), 'Engines (Scapy)') + _, report, _ = self.report('success', check_runs(GREEN)) + self.assertLessEqual(len(report), check_suite_report.LIMIT - check_suite_report.FOOTER) + self.assertIn('
Python 3.14\n\nshort\n\n
', report) + self.assertIn(f'\n\n*Trimmed to fit the comment; the whole summary is on [the run]({RUN_URL}).*' + '\n\n
\n', report) + comment = discussion_report.compose(report, 'Posted from the run.', 'success') + self.assertNotIn('cut short', comment) + + def test_no_summary_is_no_section_and_no_warning(self) -> None: + _, report, stdout = self.report('success', check_runs(GREEN)) + self.assertNotIn('Job summaries', report) + self.assertNotIn('::warning', stdout) + + def test_unreadable_summaries_warn(self) -> None: + outputs, report, stdout = self.report('success', check_runs(GREEN), FAKE_ARTIFACTS='fail') + self.assertEqual(outputs['post'], 'true') + self.assertIn('The job summaries could not be read: `HTTP 403: Resource not accessible by integration`.', + report) + self.assertIn('::warning title=CI report not written::The job summaries could not be read', stdout) + + +class TestTheDownload(unittest.TestCase): + """:func:`check_suite_report.download` reads each artifact's folder, and one unpacked in place.""" + + def test_folders_and_the_directory_itself(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + (root / 'job-summary.md').write_text('lone\n', encoding='utf-8') + (root / 'job-summary-a-0').mkdir() + (root / 'job-summary-a-0' / 'job-summary.md').write_text('first\n', encoding='utf-8') + (root / 'job-summary-a-0' / 'job-summary.name').write_text('Python 3.14\nignored\n', encoding='utf-8') + (root / 'job-summary-b-0').mkdir() + (root / 'job-summary-b-0' / 'job-summary.md').write_text(' \n', encoding='utf-8') + done = subprocess.CompletedProcess([], 0, '', '') + with unittest.mock.patch.object(check_suite_report.subprocess, 'run', return_value=done) as run: + found = check_suite_report.download('o/r', RUN_ID, root) + self.assertEqual(found, ([(root.name, 'lone\n'), ('Python 3.14', 'first\n')], '')) + self.assertEqual(run.call_args[0][0][-2:], ['--dir', str(root)]) + self.assertEqual(check_suite_report.download('o/r', '1; rm', root), ([], "the run id '1; rm' is not a number")) + + +class TestTheFit(unittest.TestCase): + """:func:`check_suite_report.fit` shares the room smallest first.""" + + URL = RUN_URL + + def test_the_limits_agree(self) -> None: + self.assertEqual(check_suite_report.LIMIT, discussion_report.LIMIT) + + def test_all_fit_whole(self) -> None: + blocks = check_suite_report.fit([('a', 'one'), ('b', 'two')], 1000, self.URL) + self.assertEqual(blocks, ['
a\n\none\n\n
', + '
b\n\ntwo\n\n
']) + + def test_a_short_one_stays_whole_beside_long_ones(self) -> None: + texts = [('long', 'x\n' * 5000), ('short', 'y'), ('longer', 'z\n' * 8000)] + for room in (3000, 9000, 20000): + with self.subTest(room=room): + blocks = check_suite_report.fit(texts, room, self.URL) + self.assertLessEqual(sum(map(len, blocks)), room) + self.assertEqual(len(blocks), 3) + self.assertIn('\n\ny\n\n', blocks[1]) + for block in (blocks[0], blocks[2]): + self.assertIn('*Trimmed to fit the comment', block) + + def test_a_cut_closes_its_code_fence(self) -> None: + [block] = check_suite_report.fit([('job', '```\n' + 'line\n' * 1000 + '```\n')], 600, self.URL) + body = block.split('\n\n', 1)[1].split('\n\n*Trimmed', 1)[0] + self.assertEqual(body.count('```') % 2, 0) + + def test_no_room_leaves_a_summary_out_and_says_so(self) -> None: + self.assertEqual(check_suite_report.fit([('job', 'x' * 1000)], 50, self.URL), []) + found = check_suite_report.Assessment('success', [], [], False) + text = check_suite_report.render('Unit Tests', 'push', 'success', SHA, RUN_URL, found, [], False, + [('job', 'x' * 1000)] * 400, '') + self.assertLessEqual(len(text), check_suite_report.LIMIT - check_suite_report.FOOTER) + self.assertRegex(text, r'\*\d+ job summaries did not fit; the run holds them\.\*$') #: A stand-in ``python3`` for the post step, recording its arguments and environment. @@ -421,25 +801,27 @@ class TestThePostStep(unittest.TestCase): """``Post the report to Discussions``, with a stand-in ``python3``.""" def test_it_passes_the_relayed_run(self) -> None: - with tempfile.TemporaryDirectory(prefix='pcapkit-2030-push-post-') as tmp: - root = pathlib.Path(tmp) - (root / 'bin').mkdir() - (root / 'work').mkdir() - write_launcher(root / 'bin', 'python3', FAKE_PYTHON) - env = {'PATH': f'{root / "bin"}{os.pathsep}{os.environ.get("PATH", "")}', - 'FAKE_PYTHON_LOG': str(root / 'log.json'), 'REPORT_CATEGORY': CATEGORY, 'TITLE': TITLE, - 'STATUS': 'warning', 'RUN_URL': RUN_URL, 'HEAD_SHA': SHA, 'GITHUB_WORKFLOW': 'Coverage Comment'} - result = run_step('Post the report to Discussions', root / 'work', env) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - logged = json.loads((root / 'log.json').read_text(encoding='utf-8')) - self.assertEqual(logged, {'workflow': 'Unit Tests', 'args': [ - 'util/discussion_report.py', '--category', CATEGORY, '--title', TITLE, '--report', REPORT, - '--mode', 'push', '--status', 'warning', '--run-url', RUN_URL, '--sha', SHA]}) + for relay in REPORTED: + with self.subTest(relay=relay), tempfile.TemporaryDirectory(prefix='pcapkit-2030-push-post-') as tmp: + root = pathlib.Path(tmp) + (root / 'bin').mkdir() + (root / 'work').mkdir() + write_launcher(root / 'bin', 'python3', FAKE_PYTHON) + env = {'PATH': f'{root / "bin"}{os.pathsep}{os.environ.get("PATH", "")}', + 'FAKE_PYTHON_LOG': str(root / 'log.json'), 'REPORT_CATEGORY': CATEGORY, 'TITLE': TITLE, + 'MODE': 'push', 'STATUS': 'warning', 'RUN_URL': RUN_URL, 'HEAD_SHA': SHA, + 'WORKFLOW': 'Unit Tests', 'GITHUB_WORKFLOW': 'Coverage Comment'} + result = run_step('Post the report to Discussions', root / 'work', env, relay) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + logged = json.loads((root / 'log.json').read_text(encoding='utf-8')) + self.assertEqual(logged, {'workflow': 'Unit Tests', 'args': [ + 'util/discussion_report.py', '--category', CATEGORY, '--title', TITLE, '--report', REPORT, + '--mode', 'push', '--status', 'warning', '--run-url', RUN_URL, '--sha', SHA]}) @unittest.skipIf(not BASH, 'needs bash on PATH') class TestEndToEnd(_FakeGitHub): - """Both steps and the real script: a bad push run posts, one green run after it, then silence.""" + """Both steps and the real scripts: the cadence each relay's runs post at.""" def setUp(self) -> None: super().setUp() @@ -447,46 +829,116 @@ def setUp(self) -> None: write_launcher(self.bin, 'gh', FAKE_GH) (self.bin / 'python3').symlink_to(sys.executable) self.work = self.tmp / 'work' - (self.work / 'util').mkdir(parents=True) - shutil.copy(SCRIPT, self.work / 'util') + checkout(self.work) self.answer = self.tmp / 'answer.json' - def push_run(self, conclusion: 'str', *runs: 'tuple[str, str, list[tuple[str, str]]]') -> None: + def run_of(self, workflow: 'str', event: 'str', conclusion: 'str', + *runs: 'tuple[str, str, list[tuple[str, ...]]]') -> None: self.answer.write_text(json.dumps(check_runs(*runs)), encoding='utf-8') shutil.rmtree(self.work / 'report', ignore_errors=True) output = self.tmp / 'output' output.write_text('', encoding='utf-8') env = self.env(FAKE_SUITE_ANSWER=str(self.answer), FAKE_SUITE_LOG=str(self.tmp / 'suite.log'), - GITHUB_OUTPUT=str(output), RUNNER_TEMP=str(self.tmp), GITHUB_WORKFLOW='Coverage Comment', - SUITE=SUITE, RUN_URL=RUN_URL, RUN_CONCLUSION=conclusion, HEAD_SHA=SHA, HEAD_BRANCH='main', - REPORT_CATEGORY=CATEGORY) - result = run_step('Write the run report', self.work, env) + GITHUB_OUTPUT=str(output), RUNNER_TEMP=str(self.tmp), GITHUB_WORKFLOW='CI Reports', + WORKFLOW=workflow, SUITE=SUITE, RUN_ID=RUN_ID, RUN_URL=RUN_URL, RUN_EVENT=event, + RUN_CONCLUSION=conclusion, HEAD_SHA=SHA, HEAD_BRANCH='main', REPORT_CATEGORY=CATEGORY) + result = run_step('Write the run report', self.work, env, RELAY) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) for line in output.read_text(encoding='utf-8').splitlines(): key, value = line.split('=', 1) env[key.upper()] = value - result = run_step('Post the report to Discussions', self.work, env) + # The step's `if:`. + if env['POST'] != 'true': + return + result = run_step('Post the report to Discussions', self.work, env, RELAY) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertNotIn('::warning', result.stdout) - def comments(self) -> 'list[str]': - found = [entry for entry in self.state['discussions'] if entry['title'] == TITLE] + def comments(self, title: 'str') -> 'list[str]': + found = [entry for entry in self.state['discussions'] if entry['title'] == title] return found[0]['comments'] if found else [] - def test_the_cadence(self) -> None: - self.push_run('success', GREEN) + def test_the_push_cadence(self) -> None: + self.run_of('Unit Tests', 'push', 'success', GREEN) + self.assertEqual(self.state['discussions'], []) + self.run_of('Unit Tests', 'push', 'success', GREEN, BLOCKING) self.assertEqual(self.state['discussions'], []) - self.push_run('success', GREEN, KTLO) - self.push_run('failure', GREEN, ('Python 3.13', 'FAILURE', [])) - self.push_run('success', GREEN) - self.push_run('success', GREEN) - comments = self.comments() + self.run_of('Unit Tests', 'push', 'success', GREEN, KTLO) + self.run_of('Unit Tests', 'push', 'failure', GREEN, ('Python 3.13', 'FAILURE', [])) + self.run_of('Unit Tests', 'push', 'success', GREEN) + self.run_of('Unit Tests', 'push', 'success', GREEN) + comments = self.comments(TITLE) self.assertEqual([MARKER.findall(body) for body in comments], [['warning'], ['failure'], ['success']]) self.assertIn('- `Python 3.8 (KTLO)`: ', comments[0]) self.assertIn('- `Python 3.13` (failure)', comments[1]) self.assertIn(f'Posted from [Unit Tests run]({RUN_URL}) at commit `{SHA}`, which ended `success`.', comments[2]) + def test_a_superseded_experimental_run_is_no_recovery(self) -> None: + """After a bad report, a run whose legs all skipped posts no "back to green"; a tested one does.""" + title = 'experimental: main push runs' + leg = 'Python 3.15t (experimental)' + self.run_of('Experimental Tests', 'push', 'success', (leg, 'SUCCESS', [('WARNING', 'tests failed')])) + self.run_of('Experimental Tests', 'push', 'success', ('Check the commit is still current', 'SUCCESS', []), + (leg, 'SKIPPED', [])) + self.assertEqual([MARKER.findall(body) for body in self.comments(title)], [['warning']]) + self.run_of('Experimental Tests', 'push', 'success', (leg, 'SUCCESS', [])) + self.assertEqual([MARKER.findall(body) for body in self.comments(title)], [['warning'], ['success']]) + + def test_a_scheduled_compatibility_run_posts_every_time(self) -> None: + title = 'python-compatibility: main scheduled runs' + for _ in range(2): + self.run_of('Python Compatibility', 'schedule', 'success', ('Compat Python 3.14', 'SUCCESS', [])) + comments = self.comments(title) + self.assertEqual([MARKER.findall(body) for body in comments], [['success'], ['success']]) + self.assertIn('Posted from [Python Compatibility run]', comments[0]) + + +class TestTheHelperCopies(unittest.TestCase): + """util/_actions.py writes the job's copy of its summary where ``JOB_SUMMARY_COPY`` names one.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.page = pathlib.Path(tmp.name) / 'summary' + self.copy = pathlib.Path(tmp.name) / 'job-summary.md' + self.copy.write_text('earlier step\n', encoding='utf-8') + + def test_both_writers_copy(self) -> None: + env = {'GITHUB_STEP_SUMMARY': str(self.page), _actions.SUMMARY_COPY: str(self.copy)} + self.assertEqual(_actions.SUMMARY_COPY, 'JOB_SUMMARY_COPY') + self.assertTrue(_actions.summary('- one', environ=env)) + with _actions.open_summary(environ=env) as page: + print('| table |', file=page) + page.write('tail\n') + self.assertEqual(self.page.read_text(encoding='utf-8'), '- one\n| table |\ntail\n') + self.assertEqual(self.copy.read_text(encoding='utf-8'), 'earlier step\n- one\n| table |\ntail\n') + + def test_without_it_nothing_is_copied(self) -> None: + env = {'GITHUB_STEP_SUMMARY': str(self.page), _actions.SUMMARY_COPY: ''} + _actions.summary('- one', environ=env) + with _actions.open_summary(environ={'GITHUB_STEP_SUMMARY': str(self.page)}) as page: + print('two', file=page) + self.assertEqual(self.page.read_text(encoding='utf-8'), '- one\ntwo\n') + self.assertEqual(self.copy.read_text(encoding='utf-8'), 'earlier step\n') + + def test_a_copy_that_cannot_open_is_left_out(self) -> None: + env = {'GITHUB_STEP_SUMMARY': str(self.page), _actions.SUMMARY_COPY: str(self.copy.parent / 'no' / 'file')} + self.assertTrue(_actions.summary('- one', environ=env)) + self.assertEqual(self.page.read_text(encoding='utf-8'), '- one\n') + + def test_the_scripts_copy_as_the_steps_run_them(self) -> None: + """``KTLO summary``'s script, run with the step's environment, writes both files.""" + env = {key: value for key, value in os.environ.items() if not key.startswith(('GITHUB_', 'PYTHON'))} + env.update(GITHUB_STEP_SUMMARY=str(self.page), JOB_SUMMARY_COPY=str(self.copy), + LEG='Python 3.8 (KTLO, non-blocking)', STEPS=json.dumps({'tests': {'outcome': 'failure'}})) + result = subprocess.run([sys.executable, 'util/leg_summary.py', 'ktlo'], cwd=ROOT, env=env, # nosec: B603 + capture_output=True, text=True, check=False, timeout=60) + self.assertEqual(result.returncode, 0, result.stderr) + line = '- Python 3.8 (KTLO, non-blocking): step `tests` failed; non-blocking, so this job still succeeds.\n' + self.assertEqual(self.page.read_text(encoding='utf-8'), line) + self.assertEqual(self.copy.read_text(encoding='utf-8'), 'earlier step\n' + line) + if __name__ == '__main__': unittest.main() diff --git a/tests/project/test_workflow_scripts_1967.py b/tests/project/test_workflow_scripts_1967.py index 236873a08d..a7e85d7561 100644 --- a/tests/project/test_workflow_scripts_1967.py +++ b/tests/project/test_workflow_scripts_1967.py @@ -225,7 +225,8 @@ def test_the_step_runs_the_script(self) -> None: if step.get('name') == 'Report coverage') self.assertEqual(step['run'], 'python util/coverage_report.py') self.assertEqual(step['shell'], 'bash') - self.assertEqual(set(step['env']), {'SHA', 'PYTHON', 'TESTS_OUTCOME'}) + # #2030: and the job's copy of its summary. + self.assertEqual(set(step['env']), {'SHA', 'PYTHON', 'TESTS_OUTCOME', 'JOB_SUMMARY_COPY'}) self.assertTrue(re.search(r"steps\.tests\.outcome", step['env']['TESTS_OUTCOME'])) diff --git a/tests/project/test_workflows_at_a_glance_1732.py b/tests/project/test_workflows_at_a_glance_1732.py index 9c801c60f4..be875c74ba 100644 --- a/tests/project/test_workflows_at_a_glance_1732.py +++ b/tests/project/test_workflows_at_a_glance_1732.py @@ -163,7 +163,10 @@ def test_a_missing_row_is_caught(self) -> None: node = re.search(r'(?m)^ +\w+\["[^"<]+
experimental\.yml"\]\n', page) self.assertIsNotNone(node) assert node is not None - self.assertEqual(problems(page.replace(node.group(0), ''), FILES), ['no graph node for experimental.yml']) + # Each graph that draws it defines it, the second for CI Reports' edge (#2030), which then dangles too. + self.assertEqual(problems(page.replace(node.group(0), ''), FILES), + ["ci-reports.yml: no graph edge 'None ==>|workflow_run: completed| RP'", + 'no graph node for experimental.yml']) if __name__ == '__main__': diff --git a/util/_actions.py b/util/_actions.py index cb81c79aae..7063ba53c5 100644 --- a/util/_actions.py +++ b/util/_actions.py @@ -20,6 +20,14 @@ a command's message escapes ``%``, ``\\r`` and ``\\n``, and each property's value escapes ``:`` and ``,`` as well. +A step that sets ``JOB_SUMMARY_COPY`` to a file has :func:`summary` and +:func:`open_summary` append what they write to that file too, the job's copy of +its summary (#2030). GitHub serves no job summary through its API, so the jobs +whose runs :file:`util/check_suite_report.py` reports upload the copy as an +artifact for the report to fold in. Only a step that writes the summary sets it, +never one that runs the test suite, whose tests write summaries of their own +through this module. + """ from __future__ import annotations @@ -34,11 +42,13 @@ from os import PathLike from typing import IO, Iterator, Mapping, Optional, Sequence, Tuple, Union -__all__ = ['LEVELS', 'escape_data', 'escape_property', 'annotation', 'annotate', 'group', +__all__ = ['LEVELS', 'SUMMARY_COPY', 'escape_data', 'escape_property', 'annotation', 'annotate', 'group', 'set_output', 'summary', 'open_summary', 'emit', 'say', 'exit_status', 'run'] #: The annotation commands, by level. LEVELS = ('error', 'warning', 'notice') +#: The variable naming the job's copy of its summary (#2030). +SUMMARY_COPY = 'JOB_SUMMARY_COPY' def escape_data(text: 'str') -> 'str': @@ -118,22 +128,88 @@ def set_output(name: 'str', value: 'str', *, required: 'bool' = True, return True +class _Teed: + """The summary, open to append to, whose writes go to the job's copy of it too (#2030). + + The copy is best effort: once a write to it fails it is closed and left + behind, and the summary is written all the same. + + """ + + def __init__(self, page: 'IO[str]', copy: 'IO[str]') -> 'None': + self._page = page + self._copy = copy # type: Optional[IO[str]] + + def _drop(self) -> 'None': + copy, self._copy = self._copy, None + if copy is not None: + with contextlib.suppress(OSError): + copy.close() + + def write(self, text: 'str') -> 'int': + """Write ``text`` to the summary and to the copy; the characters written to the summary.""" + written = self._page.write(text) + if self._copy is not None: + try: + self._copy.write(text) + except OSError: + self._drop() + return written + + def flush(self) -> 'None': + """Flush the summary and the copy.""" + self._page.flush() + if self._copy is not None: + try: + self._copy.flush() + except OSError: + self._drop() + + def close(self) -> 'None': + """Close the summary and the copy.""" + try: + self._page.close() + finally: + self._drop() + + def __enter__(self) -> '_Teed': + return self + + def __exit__(self, *_: 'object') -> 'None': + self.close() + + def open_summary(*, environ: 'Optional[Mapping[str, str]]' = None) -> 'IO[str]': - """``$GITHUB_STEP_SUMMARY``, open to append to, in UTF-8; :exc:`KeyError` where it is unset.""" - return open((os.environ if environ is None else environ)['GITHUB_STEP_SUMMARY'], 'a', encoding='utf-8') + """``$GITHUB_STEP_SUMMARY``, open to append to, in UTF-8; :exc:`KeyError` where it is unset. + + Where ``$JOB_SUMMARY_COPY`` names a file that opens, what is written goes + there too; one that does not open is left out. + + """ + environ = os.environ if environ is None else environ + # Both are the caller's to close, as the file this returned before #2030 was. + page = open(environ['GITHUB_STEP_SUMMARY'], 'a', encoding='utf-8') # pylint: disable=consider-using-with + path = environ.get(SUMMARY_COPY) + if not path: + return page + try: + copy = open(path, 'a', encoding='utf-8') # pylint: disable=consider-using-with + except OSError: + return page + return _Teed(page, copy) # type: ignore[return-value] def summary(*lines: 'str', required: 'bool' = True, environ: 'Optional[Mapping[str, str]]' = None) -> 'bool': """Append each of ``lines``, and a newline, to ``$GITHUB_STEP_SUMMARY``; whether it was written. Where ``$GITHUB_STEP_SUMMARY`` is unset, a ``required`` write raises - :exc:`KeyError` and any other writes nothing. + :exc:`KeyError` and any other writes nothing. The lines go to + ``$JOB_SUMMARY_COPY`` too, as :func:`open_summary` writes. """ - path = _path('GITHUB_STEP_SUMMARY', required, environ) - if path is None: + if _path('GITHUB_STEP_SUMMARY', required, environ) is None: return False - with open(path, 'a', encoding='utf-8') as file: + with open_summary(environ=environ) as file: for text in lines: file.write(text + '\n') return True diff --git a/util/check_suite_report.py b/util/check_suite_report.py new file mode 100644 index 0000000000..80564d87b5 --- /dev/null +++ b/util/check_suite_report.py @@ -0,0 +1,397 @@ +# -*- coding: utf-8 -*- +"""Write a test workflow run's report from its check suite and its job summaries (#2030). + +The ``discussion`` job of :file:`.github/workflows/coverage-comment.yml`, for +Unit Tests, and of :file:`.github/workflows/ci-reports.yml`, for Experimental +Tests and Python Compatibility, runs, from a checkout of the default branch's +script and on the runner's ``python3``:: + + python3 util/check_suite_report.py --report FILE + +with ``GH_TOKEN`` and ``GITHUB_REPOSITORY``, and the triggering run's +``workflow_run`` fields, in the environment: ``WORKFLOW``, its workflow's name, +which :data:`WORKFLOWS` must hold; ``SUITE``, its check suite's node id; +``RUN_ID``, ``RUN_URL``, ``RUN_EVENT``, ``RUN_CONCLUSION``, ``HEAD_SHA`` and +``HEAD_BRANCH``. It writes the Markdown report to ``FILE``, and the step outputs +``post``, ``title``, ``mode`` and ``status`` that ``Post the report to +Discussions`` hands :file:`util/discussion_report.py`. + +The run's jobs are read in one GraphQL call (:data:`SUITE_QUERY`): each job's +conclusion and its first annotations. A job that did not pass is named. A job +that passed is named with its warning and failure annotations, other than those +:func:`ignored` drops: pip's ``Retrying ... after connection`` lines, network +noise on any leg, and the annotations a workflow's ``expected`` list holds as +written on every run by design. Each name and annotation is quoted as a code span, +so none renders or mentions. + +The status (:func:`assess`) starts from the run's conclusion (:data:`CONCLUSIONS`). +A green run becomes ``warning`` where a job failed without failing it, and where +an annotation counts. On a push run, the owner's ruling, one counts only on a +non-blocking leg (:func:`nonblocking`), whose steps are forced to succeed: a +``(KTLO)`` or ``(experimental)`` job, or an annotation such a step titles +``(KTLO, non-blocking)``. A passing blocking job's annotations then neither count +nor are listed, unless the run failed. On any other run, which posts every time, +every annotation counts. + +A run whose legs, the jobs a workflow's ``legs`` pattern names, all skipped, as +they do where ``current`` or ``changes``' ``code`` is ``false``, tested nothing, +so where it did not fail it posts nothing (``post=false``). + +Each job's summary comes from the artifact ``job-summary-*`` the job uploads +(:func:`download`), since the API serves none: ``job-summary.md``, the job's +copy of its summary, and ``job-summary.name``, the job's name. Each is folded +into a ``
`` block, and the blocks share what room a comment leaves +(:func:`fit`), each trimmed to its share with a link to the run. + +Reporting must never turn a run red: the script exits 0 whatever happens, and +where anything goes wrong it writes a ``::warning`` and the report it can. It is +standard library only, and reaches the API only through :program:`gh`. + +""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import pathlib +import re +# The gh CLI, with a fixed argument list and no shell. +import subprocess # nosec: B404 +import sys +import tempfile +from typing import TYPE_CHECKING, NamedTuple + +try: # #2047: run as `python util/