From 76a7ea434e451eb3efc9b72f0e28fedb39602051 Mon Sep 17 00:00:00 2001 From: dcouthier Date: Tue, 2 Dec 2025 11:51:28 +0100 Subject: [PATCH] feat: update ci process --- .github/workflows/sanity-check.yml | 22 ++++++---- CI/check_model.py | 13 +++--- CI/post_filter.py | 69 ++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 14 deletions(-) create mode 100644 CI/post_filter.py diff --git a/.github/workflows/sanity-check.yml b/.github/workflows/sanity-check.yml index dea7924..d33bf7a 100644 --- a/.github/workflows/sanity-check.yml +++ b/.github/workflows/sanity-check.yml @@ -34,19 +34,25 @@ jobs: id: find_files run: | FILES=$(git diff --diff-filter=A --name-only origin/main...HEAD | grep "_i8_" || true) - { - echo "files<> $GITHUB_OUTPUT + fi + echo "count=$COUNT" >> $GITHUB_OUTPUT + echo "files<> $GITHUB_OUTPUT + echo "$FILES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Run model checks - if: steps.find_files.outputs.files != '' + if: steps.find_files.outputs.count != '0' run: | while read -r f; do [ -z "$f" ] && continue echo "Checking model: $f" git lfs pull --include="$f" --exclude="" - python CI/check_model.py --model "$f" - done <<< "${{ steps.find_files.outputs.files }}" - + python CI/post_filter.py -- CI/check_model.py --models "$f" + done <<< "${{ steps.find_files.outputs.files }}" \ No newline at end of file diff --git a/CI/check_model.py b/CI/check_model.py index fe7b0ad..c935b9b 100644 --- a/CI/check_model.py +++ b/CI/check_model.py @@ -1,12 +1,10 @@ import argparse +import os -import akida from cnn2snn import convert from quantizeml import load_model - from compute_device import compute_min_device - def process_model(file_path): try: print(f"Loading model: {file_path}") @@ -29,8 +27,11 @@ def process_model(file_path): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--model", required=True, help="Path to model file (.h5 or .onnx)") + parser.add_argument("--models", nargs="+", required=True, + help="Path to model files (.h5 or .onnx)") args = parser.parse_args() - process_model(args.model) - + # Process each model + for model_file in args.models: + if os.path.exists(model_file): + process_model(model_file) diff --git a/CI/post_filter.py b/CI/post_filter.py new file mode 100644 index 0000000..837d1f5 --- /dev/null +++ b/CI/post_filter.py @@ -0,0 +1,69 @@ +"""Simple post-capture filter for noisy commands. +Runs a subprocess, captures stdout+stderr, and prints allowed lines to the console. +Usage: + python CI/post_filter.py -- CI/check_model.py --models $FILES + The `--` separates options to this script from the command to run. +""" +import os +import sys +import argparse +import subprocess +import re + +def run_and_filter(cmd:str): + """Runs a subprocess, captures stdout+stderr, and prints filtred lines to the console. + Args: + cmd (str): Python command sent from GitHub action + """ + allow = re.compile(r'(Checking model:|Loading model:|✅.*needs \d+ Akida nodes)', re.IGNORECASE) + deny = re.compile( + r'(cuda|cufft|cudnn|cublas|tensorflow|xla|absl::InitializeLog|Unable to register|WARNING:|' + r'failed call to|loop_optimizer\.cc|computation_placer|computation placer|gpu_device|Created device|' + r'TF_FORCE_GPU_ALLOW_GROWTH|Overriding orig_value)' + , re.IGNORECASE) + + # If the command is a Python script path and not executable, prefix + # with the current Python interpreter to avoid PermissionError on CI. + if isinstance(cmd, (list, tuple)) and cmd: + first = cmd[0] + if isinstance(first, str) and first.endswith('.py') and not os.access(first, os.X_OK): + cmd = [sys.executable] + list(cmd) + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + + try: + # Read lines as they arrive and print + for raw in iter(proc.stdout.readline, ''): + if raw is None: + break + line = raw.rstrip('\n') + keep = allow.search(line) or not deny.search(line) + if keep: + print(line, flush=True) + + proc.wait() + finally: + try: + proc.stdout.close() + except Exception: + pass + +def main(argv=None): + p = argparse.ArgumentParser(prog='post_filter', description='Run command and filter its output') + p.add_argument('cmd', nargs=argparse.REMAINDER, help='Command to run (use -- before the command)') + args = p.parse_args(argv) + + if not args.cmd: + print('No command provided.' + 'Use -- to separate command, e.g.: python CI/post_filter.py -- CI/check_model.py --models $FILES') + return 2 + + # Remove leading '--' if present + cmd = args.cmd + if cmd and cmd[0] == '--': + cmd = cmd[1:] + + run_and_filter(cmd) + +if __name__ == '__main__': + sys.exit(main())