Skip to content

refactor(util): move ci_timing and vendor_test_gate onto the Actions helper (#2047) #2063

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JarryShaw merged 1 commit into from
Oct 23, 2026
Merged

refactor(util): move ci_timing and vendor_test_gate onto the Actions helper (#2047) #2063

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions tests/project/test_actions_helper_2047.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,22 +40,24 @@
ROOT = pathlib.Path(__file__).resolve().parents[2]
HELPER = ROOT / 'util' / '_actions.py'
#: The scripts that import the helper.
SCRIPTS = ('apt_packages', 'classify_changes', 'codeql_report', 'coverage_report', 'leg_summary', 'pip_install',
'regenerate_samples', 'vendor_code_changed')
SCRIPTS = ('apt_packages', 'ci_timing', 'classify_changes', 'codeql_report', 'coverage_report', 'leg_summary',
'pip_install', 'regenerate_samples', 'vendor_code_changed', 'vendor_test_gate')
#: The top-level modules the helper may import, each in Python 3.7's standard
#: library: the running interpreter's :data:`sys.stdlib_module_names` would
#: pass ``tomllib``, which 3.7 lacks.
STDLIB_37 = frozenset({'__future__', 'contextlib', 'os', 'subprocess', 'sys', 'typing', 'uuid'})
#: A trivial run of each script, ``(arguments, environment)``, that gets past its imports.
RUNS = {
'apt_packages': (['--help'], {}),
'ci_timing': (['--help'], {}),
'classify_changes': ([], {'OWN_PR': 'false'}),
'codeql_report': (['--help'], {}),
'coverage_report': (['--help'], {}),
'leg_summary': (['ktlo'], {'LEG': 'Python 3.8 (KTLO, non-blocking)', 'STEPS': '{}'}),
'pip_install': (['--help'], {}),
'regenerate_samples': (['--help'], {}),
'vendor_code_changed': (['--help'], {}),
'vendor_test_gate': (['--help'], {}),
}
#: ``-X importtime``'s line for a module, by its name.
IMPORT_TIME = re.compile(r'^import time:.*\|\s*(\S+)\s*$', re.M)
Expand Down
19 changes: 12 additions & 7 deletions util/ci_timing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,11 @@

import requests

try: # #2047: run as `python util/<script>.py`, with util/ first on sys.path
import _actions
except ImportError: # imported as util.<script>, or loaded by path, from the repository root
from util import _actions

if TYPE_CHECKING:
from typing import Any, Callable, Iterable, Iterator, Optional

Expand Down Expand Up @@ -1365,8 +1370,8 @@ def main(argv: 'Optional[list[str]]' = None, *, transport: 'Optional[Transport]'

token = os.environ.get('GH_TOKEN') or os.environ.get('GITHUB_TOKEN')
if not args.repo or not (token or transport):
print('::error title=CI timing::Needs --repo (or $GITHUB_REPOSITORY) and a token in '
'$GH_TOKEN or $GITHUB_TOKEN.')
_actions.annotate('error', 'Needs --repo (or $GITHUB_REPOSITORY) and a token in $GH_TOKEN or $GITHUB_TOKEN.',
title='CI timing')
return EXIT_API

client = Client(token or '', transport=transport, max_calls=args.max_calls)
Expand All @@ -1381,13 +1386,13 @@ def main(argv: 'Optional[list[str]]' = None, *, transport: 'Optional[Transport]'
budget=args.budget_minutes * 60, clock=clock,
progress=lambda line: print(line, file=sys.stderr, flush=True))
except (BudgetExceeded, OSError) as error:
print(f'::error title=CI timing could not read the API::{error}. '
'No threshold was evaluated.')
_actions.annotate('error', f'{error}. No threshold was evaluated.',
title='CI timing could not read the API', escape=False)
return EXIT_API
return _report(args, client, data)
except Exception: # pylint: disable=broad-except
traceback.print_exc()
print('::error title=CI timing crashed::See the traceback above. No threshold was evaluated.')
_actions.annotate('error', 'See the traceback above. No threshold was evaluated.', title='CI timing crashed')
return EXIT_CRASH


Expand Down Expand Up @@ -1415,11 +1420,11 @@ def _report(args: 'argparse.Namespace', client: 'Client', data: 'dict[str, Any]'
file.write(text)
print(text)
if data.get('stopped'):
print(f"::warning title=CI timing stopped early::{data['stopped']}")
_actions.annotate('warning', data['stopped'], title='CI timing stopped early', escape=False)
for item in findings:
kind = 'error' if item['level'] == 'hard' else 'warning'
message = item['message'].replace('`', '').replace('%', '%25')
print(f'::{kind} title=CI timing {item["id"]}::{message}')
_actions.annotate(kind, message, title=f'CI timing {item["id"]}', escape=False)
return EXIT_HARD if any(item['level'] == 'hard' for item in findings) else 0


Expand Down
63 changes: 28 additions & 35 deletions util/vendor_test_gate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@
import time
from typing import TYPE_CHECKING, NamedTuple

try: # #2047: run as `python util/<script>.py`, with util/ first on sys.path
import _actions
except ImportError: # imported as util.<script>, or loaded by path, from the repository root
from util import _actions

try:
import psutil
except ImportError: # the test extra installs it on CPython only
Expand Down Expand Up @@ -218,11 +223,6 @@ def _ps_group_rss(pgid: 'int') -> 'Optional[int]':
return total


def escape(text: 'str') -> 'str':
"""``text`` as the data of a workflow command."""
return text.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A')


class Gate:
"""The test command, run on some of ``changes`` at a time."""

Expand Down Expand Up @@ -278,21 +278,20 @@ def cap() -> None:

cache = tempfile.mkdtemp(prefix='vendor-test-gate-')
env['PYTHONPYCACHEPREFIX'] = cache
print(f'::group::{label}', flush=True)
try:
child = subprocess.Popen(self.command, env=env, # nosec B603
start_new_session=True, preexec_fn=cap)
try:
stopped = self._watch(child)
finally:
# Whatever the run left running in its session goes with it,
# and the run itself is reaped, however this was left.
with contextlib.suppress(ProcessLookupError, PermissionError):
os.killpg(child.pid, signal.SIGKILL)
with contextlib.suppress(subprocess.TimeoutExpired):
child.wait(timeout=GRACE)
with _actions.group(label):
child = subprocess.Popen(self.command, env=env, # nosec B603
start_new_session=True, preexec_fn=cap)
try:
stopped = self._watch(child)
finally:
# Whatever the run left running in its session goes with it,
# and the run itself is reaped, however this was left.
with contextlib.suppress(ProcessLookupError, PermissionError):
os.killpg(child.pid, signal.SIGKILL)
with contextlib.suppress(subprocess.TimeoutExpired):
child.wait(timeout=GRACE)
finally:
print('::endgroup::', flush=True)
shutil.rmtree(cache, ignore_errors=True)
if stopped:
return stopped
Expand Down Expand Up @@ -374,36 +373,30 @@ def report(changes: 'Sequence[Change]', kept: 'FrozenSet[str]', reason: 'str', g
"HEAD's copy of it stays, as #872 does for a sub-vendor that raised. The other "
'regenerated modules passed and go on to be committed.')
for path in discarded:
print(f'::warning title=Vendor test gate discarded a module::{escape(path)}: '
f"{escape(command)} fails with it on top of HEAD's tree, so HEAD's copy is kept "
'and the rest of the regeneration is committed; see the step summary')
_actions.annotate('warning', f"{path}: {command} fails with it on top of HEAD's tree, so "
"HEAD's copy is kept and the rest of the regeneration is committed; "
'see the step summary', title='Vendor test gate discarded a module')
else:
heading = 'discarded every regenerated module'
body = (f"{reason}. HEAD's copy of every module is kept, so nothing from this crawl is "
f'committed. The tests are `{command}`.')
print(f'::warning title=Vendor test gate discarded every regenerated module::'
f'{escape(reason)}; see the step summary')
_actions.annotate('warning', f'{reason}; see the step summary',
title='Vendor test gate discarded every regenerated module')
lines = [f'### :warning: Vendor test gate {heading}', '', body, '',
'| Module | Kept |', '| --- | --- |']
lines.extend(f'| `{change.path}` | {"yes" if change.path in kept else "no"} |' for change in changes)
lines.extend(['', '| Run | Modules applied | Result | Seconds |', '| --- | --- | --- | --- |'])
lines.extend(f'| {run.number} | {run.modules} of {len(changes)} | {run.result} | {run.seconds:.0f} |'
for run in gate.runs)
if gate.unsampled:
print(f'::warning title=Vendor test gate could not sample memory::{SAMPLER} could not read '
"the run's memory, so the memory cap was not enforced on every sample")
summary = os.environ.get('GITHUB_STEP_SUMMARY')
if summary:
with open(summary, 'a', encoding='utf-8') as file:
print('\n'.join(lines), file=file)
_actions.annotate('warning', f"{SAMPLER} could not read the run's memory, so the memory cap "
'was not enforced on every sample', title='Vendor test gate could not sample memory')
_actions.summary(*lines, required=False)
return verdict


def write_verdict(verdict: 'str') -> None:
output = os.environ.get('GITHUB_OUTPUT')
if output:
with open(output, 'a', encoding='utf-8') as file:
print(f'verdict={verdict}', file=file)
_actions.set_output('verdict', verdict, required=False)


def _terminate(signum: 'int', frame: 'object') -> 'NoReturn': # pylint: disable=unused-argument
Expand Down Expand Up @@ -452,8 +445,8 @@ def main(argv: 'Optional[list[str]]' = None) -> int:
strays = [path for path in changed_modules() if path not in set(paths)]
apply(snapshot(strays), frozenset())
for path in strays:
print(f'::warning title=Vendor test gate restored a module::{escape(path)} changed '
'during the test runs, so it is put back as HEAD has it')
_actions.annotate('warning', f'{path} changed during the test runs, so it is put back as HEAD has it',
title='Vendor test gate restored a module')
except BaseException:
apply(changes, frozenset())
raise
Expand Down
Loading