From c2998c9f3943bb38391bd5388d1e9c962f52292a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9sar=20Miguel=20Valdez=20C=C3=B3rdova?= Date: Thu, 1 Oct 2026 15:43:03 -0400 Subject: [PATCH 1/5] Add workshop material lint workflow Checks the files a pull request changes: notebooks must be valid JSON, Python, R and shell scripts must parse, Rmd/qmd headers and chunks must be well formed. Those fail the check. Code cells that do not parse, committed error outputs and relative image paths are reported as warnings, since student notebooks contain intentional blanks. Runs on pull_request with a read-only token, separate from the checklist workflow. Co-Authored-By: Claude Fable 5.1 --- .github/scripts/lint_workshop.py | 344 ++++++++++++++++++++++++++++ .github/workflows/lint_workshop.yml | 50 ++++ 2 files changed, 394 insertions(+) create mode 100644 .github/scripts/lint_workshop.py create mode 100644 .github/workflows/lint_workshop.yml diff --git a/.github/scripts/lint_workshop.py b/.github/scripts/lint_workshop.py new file mode 100644 index 0000000..d22c5e2 --- /dev/null +++ b/.github/scripts/lint_workshop.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Lint workshop material. + +ERROR = the file cannot be opened or run at all (fails the check). +WARN = likely mistake, but could be intentional in teaching material. + +Usage: + lint_workshop.py FILE... lint the given files + lint_workshop.py --changed-since REF lint files changed between REF and HEAD + lint_workshop.py --all lint every tracked file +Add --github for GitHub annotations and a job summary, --print-files to only +list what would be linted. +""" +import argparse +import ast +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import warnings +import zipfile +from pathlib import Path + +try: + import nbformat +except ImportError: # JSON check still runs without it + nbformat = None +try: + import yaml +except ImportError: + yaml = None + +RSCRIPT = shutil.which("Rscript") +BASH = shutil.which("bash") +SKIPPED = set() # checks that could not run because a tool is missing + +# Parses every file given on the command line, prints "pathmessage" for failures. +R_PARSE = ( + "for (f in commandArgs(TRUE)) tryCatch(invisible(parse(file = f, keep.source = FALSE))," + " error = function(e) cat(f, '\\t', gsub('\\n', ' ', conditionMessage(e)), '\\n', sep = ''))" +) + + +def r_parse_failures(paths): + """Return {path: message} for R files that do not parse. Empty if R is missing.""" + if not paths: + return {} + if not RSCRIPT: + SKIPPED.add("R code was not checked: Rscript is not installed") + return {} + failures = {} + for start in range(0, len(paths), 200): + batch = [str(p) for p in paths[start:start + 200]] + out = subprocess.run([RSCRIPT, "-e", R_PARSE, *batch], capture_output=True, text=True) + for line in out.stdout.splitlines(): + if "\t" in line: + path, msg = line.split("\t", 1) + failures[path] = re.sub(r"^.*?:(?=\d+:\d+:)", "", msg).strip() # drop the file path R prepends + return failures + + +def strip_ipython(source): + """Blank out IPython magics and shell escapes so the rest can go through ast.""" + if source.lstrip().startswith("%%"): + first = source.lstrip().split(None, 1)[0] + if first not in ("%%time", "%%timeit", "%%capture"): + return None # the cell body is not Python + source = source.split("\n", 1)[1] if "\n" in source else "" + lines = [] + for line in source.split("\n"): + stripped = line.lstrip() + indent = line[: len(line) - len(stripped)] + if stripped.startswith(("%", "!")) or stripped.startswith("?") or stripped.rstrip().endswith("?"): + line = indent + "pass" + else: + # `x = !ls` and `x = %magic` assignments + line = re.sub(r"=\s*[!%].*$", "= None", line) + lines.append(line) + return "\n".join(lines) + + +def python_syntax_error(source): + try: + ast.parse(source) + except SyntaxError as e: + return f"{type(e).__name__}: {e.msg} (line {e.lineno})" + return None + + +RELATIVE_IMG = re.compile(r'!\[[^\]]*\]\(([^)\s]+)|]+src=["\']([^"\']+)') + + +def lint_ipynb(path, add): + text = path.read_text(encoding="utf-8", errors="replace") + try: + nb = json.loads(text) + except json.JSONDecodeError as e: + add("ERROR", f"line {e.lineno}", f"not valid JSON, the notebook cannot be opened: {e.msg}") + return + if not isinstance(nb, dict) or "cells" not in nb and "worksheets" not in nb: + add("ERROR", "", "valid JSON but not a notebook (no cells)") + return + if nbformat is not None: + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + nbformat.validate(nb) + except Exception as e: + msg = str(getattr(e, "message", e)).split("\n")[0][:160] + add("WARN", "", f"does not match the notebook schema: {msg}") + + meta = nb.get("metadata", {}) + language = ( + meta.get("kernelspec", {}).get("language") + or meta.get("language_info", {}).get("name") + or ("R" if meta.get("kernelspec", {}).get("name", "").lower() in ("ir", "r") else "python") + ).lower() + + r_chunks = {} + tmp = Path(tempfile.mkdtemp()) + for i, cell in enumerate(nb.get("cells", [])): + source = cell.get("source", "") + source = "".join(source) if isinstance(source, list) else source + where = f"cell {i}" + if cell.get("cell_type") == "markdown": + for m in RELATIVE_IMG.finditer(source): + target = m.group(1) or m.group(2) + if not target.startswith(("http://", "https://", "data:", "attachment:", "#")): + add("WARN", where, f"relative image path does not render in Colab: {target}") + elif cell.get("cell_type") == "code": + for output in cell.get("outputs", []): + if output.get("output_type") == "error": + add("WARN", where, f"committed with an error output: {output.get('ename')}: {str(output.get('evalue'))[:80]}") + if not source.strip(): + continue + if language == "python": + cleaned = strip_ipython(source) + err = python_syntax_error(cleaned) if cleaned is not None else None + if err: + add("WARN", where, f"code cell does not parse: {err}") + elif language == "r": + chunk = tmp / f"{i}.R" + chunk.write_text(source) + r_chunks[str(chunk)] = where + for chunk, msg in r_parse_failures(list(r_chunks)).items(): + add("WARN", r_chunks[chunk], f"code cell does not parse: {msg[:140]}") + shutil.rmtree(tmp, ignore_errors=True) + + +def lint_py(path, add): + err = python_syntax_error(path.read_text(encoding="utf-8", errors="replace")) + if err: + add("ERROR", "", f"does not parse: {err}") + + +def lint_r(path, add): + for msg in r_parse_failures([path]).values(): + add("ERROR", "", f"does not parse: {msg[:160]}") + + +def lint_sh(path, add): + if not BASH: + SKIPPED.add("shell scripts were not checked: bash is not installed") + return + out = subprocess.run([BASH, "-n", str(path)], capture_output=True, text=True) + if out.returncode: + add("ERROR", "", f"bash -n: {out.stderr.strip().splitlines()[-1][:160]}") + + +FENCE = re.compile(r"^\s*(`{3,}|~{3,})(.*)$") +CHUNK_HEADER = re.compile(r"^\{\s*([A-Za-z0-9_]+)(.*)\}\s*$") +NOT_RUN = re.compile(r"eval\s*[=:]\s*(FALSE|F|false)\b|error\s*[=:]\s*(TRUE|T|true)\b") + + +def lint_rmd(path, add): + lines = path.read_text(encoding="utf-8", errors="replace").split("\n") + + # YAML front matter + if lines and lines[0].strip() == "---": + end = next((i for i in range(1, len(lines)) if lines[i].strip() in ("---", "...")), None) + if end is None: + add("ERROR", "line 1", "YAML header is never closed with ---") + elif yaml is not None: + class Loader(yaml.SafeLoader): + pass + Loader.add_multi_constructor("!", lambda loader, suffix, node: None) # !r, !expr + try: + yaml.load("\n".join(lines[1:end]), Loader=Loader) + except yaml.YAMLError as e: + add("ERROR", "line 1", f"YAML header does not parse, the document cannot render: {str(e).splitlines()[0][:140]}") + + # Code fences and chunks + chunks = [] # (start_line, language, header, code) + opened = None + for n, line in enumerate(lines, 1): + m = FENCE.match(line) + if opened is None: + if m: + opened = (n, m.group(1), m.group(2).strip(), []) + elif m and m.group(1)[0] == opened[1][0] and len(m.group(1)) >= len(opened[1]) and not m.group(2).strip(): + header = CHUNK_HEADER.match(opened[2]) + if header: + chunks.append((opened[0], header.group(1).lower(), header.group(2), "\n".join(opened[3]))) + opened = None + else: + opened[3].append(line) + if opened is not None: + add("ERROR", f"line {opened[0]}", "code chunk is never closed; everything after it renders as code") + + tmp = Path(tempfile.mkdtemp()) + r_chunks = {} + for start, language, header, code in chunks: + options = header + "\n" + "\n".join(l for l in code.split("\n") if l.startswith("#|")) + if NOT_RUN.search(options) or not code.strip(): + continue + where = f"line {start}" + if language == "r": + chunk = tmp / f"{start}.R" + chunk.write_text(code) + r_chunks[str(chunk)] = where + elif language == "python": + cleaned = strip_ipython(code) + err = python_syntax_error(cleaned) if cleaned is not None else None + if err: + add("WARN", where, f"python chunk does not parse: {err}") + elif language in ("bash", "sh") and BASH: + out = subprocess.run([BASH, "-n"], input=code, capture_output=True, text=True) + if out.returncode: + add("WARN", where, f"bash chunk does not parse: {out.stderr.strip().splitlines()[-1][:120]}") + for chunk, msg in r_parse_failures(list(r_chunks)).items(): + add("WARN", r_chunks[chunk], f"R chunk does not parse: {msg[:140]}") + shutil.rmtree(tmp, ignore_errors=True) + + +def lint_mlx(path, add): + if not zipfile.is_zipfile(path): + add("ERROR", "", "not a valid .mlx archive, MATLAB cannot open it") + + +LINTERS = { + ".ipynb": lint_ipynb, + ".py": lint_py, + ".r": lint_r, + ".rmd": lint_rmd, + ".qmd": lint_rmd, + ".sh": lint_sh, + ".mlx": lint_mlx, +} + + +def lint(paths): + findings = [] + for path in map(Path, paths): + linter = LINTERS.get(path.suffix.lower()) + if linter is None or not path.is_file(): + continue + + def add(level, where, message, path=path): + findings.append({"level": level, "file": str(path), "where": where, "message": message}) + + try: + linter(path, add) + except Exception as e: # a linter crash must not look like a clean file + add("ERROR", "", f"linter crashed: {type(e).__name__}: {e}") + return findings + + +# When the linter itself changes, every file is checked. +SELF = (".github/scripts/lint_workshop.py", ".github/workflows/lint_workshop.yml") + + +def git_files(command, *args): + out = subprocess.run(["git", command, "-z", *args], capture_output=True, text=True, check=True) + return [p for p in out.stdout.split("\0") if p] + + +def select_files(args): + if args.all: + files = git_files("ls-files") + elif args.changed_since: + files = git_files("diff", "--name-only", "--diff-filter=AMR", args.changed_since, "HEAD") + if any(f in SELF for f in files): + files = git_files("ls-files") + else: + files = args.files + return [f for f in files if Path(f).suffix.lower() in LINTERS] + + +def escape(text, is_property=False): + text = text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + return text.replace(":", "%3A").replace(",", "%2C") if is_property else text + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*") + parser.add_argument("--changed-since", metavar="REF") + parser.add_argument("--all", action="store_true") + parser.add_argument("--print-files", action="store_true") + parser.add_argument("--github", action="store_true", help="emit GitHub annotations and a job summary") + parser.add_argument("--json", help="write findings to this file") + args = parser.parse_args() + + files = select_files(args) + if args.print_files: + print("\n".join(files)) + return + + findings = lint(files) + errors = sum(f["level"] == "ERROR" for f in findings) + total = f"{len(files)} file(s) checked, {errors} error(s), {len(findings) - errors} warning(s)" + for f in findings: + text = f"{f['where']}: {f['message']}" if f["where"] else f["message"] + if args.github: + kind = "error" if f["level"] == "ERROR" else "warning" + print(f"::{kind} file={escape(f['file'], True)},title=Workshop lint::{escape(text)}") + else: + print(f"{f['level']:5} {f['file']}: {text}") + for note in sorted(SKIPPED): + print(f"::notice title=Workshop lint::{escape(note)}" if args.github else f"NOTE {note}") + print(total) + + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if args.github and summary: + lines = ["## Workshop lint", "", total, ""] + if findings: + lines += ["| | File | Where | Problem |", "|---|---|---|---|"] + for f in sorted(findings, key=lambda f: f["level"]): + problem = f["message"].replace("|", "\\|") + lines.append(f"| {'❌' if f['level'] == 'ERROR' else '⚠️'} | `{f['file']}` | {f['where']} | {problem} |") + lines += [f"- {note}" for note in sorted(SKIPPED)] + with open(summary, "a", encoding="utf-8") as out: + out.write("\n".join(lines) + "\n") + + if args.json: + Path(args.json).write_text(json.dumps(findings, indent=1)) + sys.exit(1 if errors else 0) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/lint_workshop.yml b/.github/workflows/lint_workshop.yml new file mode 100644 index 0000000..7577299 --- /dev/null +++ b/.github/workflows/lint_workshop.yml @@ -0,0 +1,50 @@ +# Checks that workshop material can be opened: notebooks are valid, scripts parse. +# Runs with a read-only token and no secrets, so it is safe for pull requests from forks. +name: Lint workshop material + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # The pull request merge commit and its first parent (the base branch) + fetch-depth: 2 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install linter dependencies + run: pip install --quiet nbformat pyyaml + + - name: Select files + id: select + env: + EVENT: ${{ github.event_name }} + run: | + if [ "$EVENT" = "pull_request" ]; then + scope="--changed-since HEAD^1" + else + scope="--all" + fi + echo "scope=$scope" >> "$GITHUB_OUTPUT" + python .github/scripts/lint_workshop.py $scope --print-files | tee files.txt + if grep -qiE '\.(r|rmd|qmd)$' files.txt; then + echo "needs_r=true" >> "$GITHUB_OUTPUT" + fi + rm files.txt + + - name: Set up R + if: steps.select.outputs.needs_r == 'true' + uses: r-lib/actions/setup-r@v2 + + - name: Lint + run: python .github/scripts/lint_workshop.py ${{ steps.select.outputs.scope }} --github From 033374dcbd261d7b0d00a922915bea8778b214ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9sar=20Miguel=20Valdez=20C=C3=B3rdova?= Date: Thu, 1 Oct 2026 15:43:55 -0400 Subject: [PATCH 2/5] TEMPORARY: notebook from #9, to confirm the lint check fails Reverted in the next commit. Co-Authored-By: Claude Fable 5.1 --- Exercises/Script/IntroToPython.ipynb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Exercises/Script/IntroToPython.ipynb b/Exercises/Script/IntroToPython.ipynb index 4d6c145..c46dfe0 100644 --- a/Exercises/Script/IntroToPython.ipynb +++ b/Exercises/Script/IntroToPython.ipynb @@ -13,10 +13,11 @@ "source": [ "# Introduction to Python\n", "\n", - "MiCM Workshop - February 18, 2026\n", + "MiCM Workshop - October 2nd, 2026\n", "\n", - "Sameena Karsan, PhD Student, Quantitative Life Sciences, McGill University\n", + "Anna Zaidi, PhD Student, Quantitative Life Sciences, McGill University\n", "\n", + "Cienna Gin-Naccarato, MSc. Student, Human Genetics, McGill University\n" "Dear `Reader | Workshop Attendee`, \n", "Welcome! In this interactive Jupyter Notebook, I will introduce you to the [Python](https://www.python.org) programming language. In this workshop, we'll journey from the beginnings of storing data in variables and doing simple calculations to writing powerful functions to perform repeatable operations.\n", "\n", From cdf01292be25eaedd65a47ef67eb971237b4c1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9sar=20Miguel=20Valdez=20C=C3=B3rdova?= Date: Thu, 1 Oct 2026 15:44:34 -0400 Subject: [PATCH 3/5] Use Node 24 versions of checkout and setup-python Co-Authored-By: Claude Fable 5.1 --- .github/workflows/lint_workshop.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_workshop.yml b/.github/workflows/lint_workshop.yml index 7577299..4296aba 100644 --- a/.github/workflows/lint_workshop.yml +++ b/.github/workflows/lint_workshop.yml @@ -13,12 +13,12 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: # The pull request merge commit and its first parent (the base branch) fetch-depth: 2 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" From 6c3003525cd5ac9c317dedbda387cb5d4e21fc4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9sar=20Miguel=20Valdez=20C=C3=B3rdova?= Date: Thu, 1 Oct 2026 15:45:09 -0400 Subject: [PATCH 4/5] Revert "TEMPORARY: notebook from #9, to confirm the lint check fails" This reverts commit 033374dcbd261d7b0d00a922915bea8778b214ef. Co-Authored-By: Claude Fable 5.1 --- Exercises/Script/IntroToPython.ipynb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Exercises/Script/IntroToPython.ipynb b/Exercises/Script/IntroToPython.ipynb index c46dfe0..4d6c145 100644 --- a/Exercises/Script/IntroToPython.ipynb +++ b/Exercises/Script/IntroToPython.ipynb @@ -13,11 +13,10 @@ "source": [ "# Introduction to Python\n", "\n", - "MiCM Workshop - October 2nd, 2026\n", + "MiCM Workshop - February 18, 2026\n", "\n", - "Anna Zaidi, PhD Student, Quantitative Life Sciences, McGill University\n", + "Sameena Karsan, PhD Student, Quantitative Life Sciences, McGill University\n", "\n", - "Cienna Gin-Naccarato, MSc. Student, Human Genetics, McGill University\n" "Dear `Reader | Workshop Attendee`, \n", "Welcome! In this interactive Jupyter Notebook, I will introduce you to the [Python](https://www.python.org) programming language. In this workshop, we'll journey from the beginnings of storing data in variables and doing simple calculations to writing powerful functions to perform repeatable operations.\n", "\n", From 4e9878568b961ebd51ec9613c84454623bd2e9f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9sar=20Miguel=20Valdez=20C=C3=B3rdova?= Date: Thu, 1 Oct 2026 15:56:04 -0400 Subject: [PATCH 5/5] Call the lint workflow from MiCM_PR_Bot instead of a local copy Test caller: points at the add-workshop-lint branch of MiCM_PR_Bot. Co-Authored-By: Claude Fable 5.1 --- .github/scripts/lint_workshop.py | 344 ---------------------------- .github/workflows/lint_workshop.yml | 42 +--- 2 files changed, 4 insertions(+), 382 deletions(-) delete mode 100644 .github/scripts/lint_workshop.py diff --git a/.github/scripts/lint_workshop.py b/.github/scripts/lint_workshop.py deleted file mode 100644 index d22c5e2..0000000 --- a/.github/scripts/lint_workshop.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 -"""Lint workshop material. - -ERROR = the file cannot be opened or run at all (fails the check). -WARN = likely mistake, but could be intentional in teaching material. - -Usage: - lint_workshop.py FILE... lint the given files - lint_workshop.py --changed-since REF lint files changed between REF and HEAD - lint_workshop.py --all lint every tracked file -Add --github for GitHub annotations and a job summary, --print-files to only -list what would be linted. -""" -import argparse -import ast -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -import warnings -import zipfile -from pathlib import Path - -try: - import nbformat -except ImportError: # JSON check still runs without it - nbformat = None -try: - import yaml -except ImportError: - yaml = None - -RSCRIPT = shutil.which("Rscript") -BASH = shutil.which("bash") -SKIPPED = set() # checks that could not run because a tool is missing - -# Parses every file given on the command line, prints "pathmessage" for failures. -R_PARSE = ( - "for (f in commandArgs(TRUE)) tryCatch(invisible(parse(file = f, keep.source = FALSE))," - " error = function(e) cat(f, '\\t', gsub('\\n', ' ', conditionMessage(e)), '\\n', sep = ''))" -) - - -def r_parse_failures(paths): - """Return {path: message} for R files that do not parse. Empty if R is missing.""" - if not paths: - return {} - if not RSCRIPT: - SKIPPED.add("R code was not checked: Rscript is not installed") - return {} - failures = {} - for start in range(0, len(paths), 200): - batch = [str(p) for p in paths[start:start + 200]] - out = subprocess.run([RSCRIPT, "-e", R_PARSE, *batch], capture_output=True, text=True) - for line in out.stdout.splitlines(): - if "\t" in line: - path, msg = line.split("\t", 1) - failures[path] = re.sub(r"^.*?:(?=\d+:\d+:)", "", msg).strip() # drop the file path R prepends - return failures - - -def strip_ipython(source): - """Blank out IPython magics and shell escapes so the rest can go through ast.""" - if source.lstrip().startswith("%%"): - first = source.lstrip().split(None, 1)[0] - if first not in ("%%time", "%%timeit", "%%capture"): - return None # the cell body is not Python - source = source.split("\n", 1)[1] if "\n" in source else "" - lines = [] - for line in source.split("\n"): - stripped = line.lstrip() - indent = line[: len(line) - len(stripped)] - if stripped.startswith(("%", "!")) or stripped.startswith("?") or stripped.rstrip().endswith("?"): - line = indent + "pass" - else: - # `x = !ls` and `x = %magic` assignments - line = re.sub(r"=\s*[!%].*$", "= None", line) - lines.append(line) - return "\n".join(lines) - - -def python_syntax_error(source): - try: - ast.parse(source) - except SyntaxError as e: - return f"{type(e).__name__}: {e.msg} (line {e.lineno})" - return None - - -RELATIVE_IMG = re.compile(r'!\[[^\]]*\]\(([^)\s]+)|]+src=["\']([^"\']+)') - - -def lint_ipynb(path, add): - text = path.read_text(encoding="utf-8", errors="replace") - try: - nb = json.loads(text) - except json.JSONDecodeError as e: - add("ERROR", f"line {e.lineno}", f"not valid JSON, the notebook cannot be opened: {e.msg}") - return - if not isinstance(nb, dict) or "cells" not in nb and "worksheets" not in nb: - add("ERROR", "", "valid JSON but not a notebook (no cells)") - return - if nbformat is not None: - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - nbformat.validate(nb) - except Exception as e: - msg = str(getattr(e, "message", e)).split("\n")[0][:160] - add("WARN", "", f"does not match the notebook schema: {msg}") - - meta = nb.get("metadata", {}) - language = ( - meta.get("kernelspec", {}).get("language") - or meta.get("language_info", {}).get("name") - or ("R" if meta.get("kernelspec", {}).get("name", "").lower() in ("ir", "r") else "python") - ).lower() - - r_chunks = {} - tmp = Path(tempfile.mkdtemp()) - for i, cell in enumerate(nb.get("cells", [])): - source = cell.get("source", "") - source = "".join(source) if isinstance(source, list) else source - where = f"cell {i}" - if cell.get("cell_type") == "markdown": - for m in RELATIVE_IMG.finditer(source): - target = m.group(1) or m.group(2) - if not target.startswith(("http://", "https://", "data:", "attachment:", "#")): - add("WARN", where, f"relative image path does not render in Colab: {target}") - elif cell.get("cell_type") == "code": - for output in cell.get("outputs", []): - if output.get("output_type") == "error": - add("WARN", where, f"committed with an error output: {output.get('ename')}: {str(output.get('evalue'))[:80]}") - if not source.strip(): - continue - if language == "python": - cleaned = strip_ipython(source) - err = python_syntax_error(cleaned) if cleaned is not None else None - if err: - add("WARN", where, f"code cell does not parse: {err}") - elif language == "r": - chunk = tmp / f"{i}.R" - chunk.write_text(source) - r_chunks[str(chunk)] = where - for chunk, msg in r_parse_failures(list(r_chunks)).items(): - add("WARN", r_chunks[chunk], f"code cell does not parse: {msg[:140]}") - shutil.rmtree(tmp, ignore_errors=True) - - -def lint_py(path, add): - err = python_syntax_error(path.read_text(encoding="utf-8", errors="replace")) - if err: - add("ERROR", "", f"does not parse: {err}") - - -def lint_r(path, add): - for msg in r_parse_failures([path]).values(): - add("ERROR", "", f"does not parse: {msg[:160]}") - - -def lint_sh(path, add): - if not BASH: - SKIPPED.add("shell scripts were not checked: bash is not installed") - return - out = subprocess.run([BASH, "-n", str(path)], capture_output=True, text=True) - if out.returncode: - add("ERROR", "", f"bash -n: {out.stderr.strip().splitlines()[-1][:160]}") - - -FENCE = re.compile(r"^\s*(`{3,}|~{3,})(.*)$") -CHUNK_HEADER = re.compile(r"^\{\s*([A-Za-z0-9_]+)(.*)\}\s*$") -NOT_RUN = re.compile(r"eval\s*[=:]\s*(FALSE|F|false)\b|error\s*[=:]\s*(TRUE|T|true)\b") - - -def lint_rmd(path, add): - lines = path.read_text(encoding="utf-8", errors="replace").split("\n") - - # YAML front matter - if lines and lines[0].strip() == "---": - end = next((i for i in range(1, len(lines)) if lines[i].strip() in ("---", "...")), None) - if end is None: - add("ERROR", "line 1", "YAML header is never closed with ---") - elif yaml is not None: - class Loader(yaml.SafeLoader): - pass - Loader.add_multi_constructor("!", lambda loader, suffix, node: None) # !r, !expr - try: - yaml.load("\n".join(lines[1:end]), Loader=Loader) - except yaml.YAMLError as e: - add("ERROR", "line 1", f"YAML header does not parse, the document cannot render: {str(e).splitlines()[0][:140]}") - - # Code fences and chunks - chunks = [] # (start_line, language, header, code) - opened = None - for n, line in enumerate(lines, 1): - m = FENCE.match(line) - if opened is None: - if m: - opened = (n, m.group(1), m.group(2).strip(), []) - elif m and m.group(1)[0] == opened[1][0] and len(m.group(1)) >= len(opened[1]) and not m.group(2).strip(): - header = CHUNK_HEADER.match(opened[2]) - if header: - chunks.append((opened[0], header.group(1).lower(), header.group(2), "\n".join(opened[3]))) - opened = None - else: - opened[3].append(line) - if opened is not None: - add("ERROR", f"line {opened[0]}", "code chunk is never closed; everything after it renders as code") - - tmp = Path(tempfile.mkdtemp()) - r_chunks = {} - for start, language, header, code in chunks: - options = header + "\n" + "\n".join(l for l in code.split("\n") if l.startswith("#|")) - if NOT_RUN.search(options) or not code.strip(): - continue - where = f"line {start}" - if language == "r": - chunk = tmp / f"{start}.R" - chunk.write_text(code) - r_chunks[str(chunk)] = where - elif language == "python": - cleaned = strip_ipython(code) - err = python_syntax_error(cleaned) if cleaned is not None else None - if err: - add("WARN", where, f"python chunk does not parse: {err}") - elif language in ("bash", "sh") and BASH: - out = subprocess.run([BASH, "-n"], input=code, capture_output=True, text=True) - if out.returncode: - add("WARN", where, f"bash chunk does not parse: {out.stderr.strip().splitlines()[-1][:120]}") - for chunk, msg in r_parse_failures(list(r_chunks)).items(): - add("WARN", r_chunks[chunk], f"R chunk does not parse: {msg[:140]}") - shutil.rmtree(tmp, ignore_errors=True) - - -def lint_mlx(path, add): - if not zipfile.is_zipfile(path): - add("ERROR", "", "not a valid .mlx archive, MATLAB cannot open it") - - -LINTERS = { - ".ipynb": lint_ipynb, - ".py": lint_py, - ".r": lint_r, - ".rmd": lint_rmd, - ".qmd": lint_rmd, - ".sh": lint_sh, - ".mlx": lint_mlx, -} - - -def lint(paths): - findings = [] - for path in map(Path, paths): - linter = LINTERS.get(path.suffix.lower()) - if linter is None or not path.is_file(): - continue - - def add(level, where, message, path=path): - findings.append({"level": level, "file": str(path), "where": where, "message": message}) - - try: - linter(path, add) - except Exception as e: # a linter crash must not look like a clean file - add("ERROR", "", f"linter crashed: {type(e).__name__}: {e}") - return findings - - -# When the linter itself changes, every file is checked. -SELF = (".github/scripts/lint_workshop.py", ".github/workflows/lint_workshop.yml") - - -def git_files(command, *args): - out = subprocess.run(["git", command, "-z", *args], capture_output=True, text=True, check=True) - return [p for p in out.stdout.split("\0") if p] - - -def select_files(args): - if args.all: - files = git_files("ls-files") - elif args.changed_since: - files = git_files("diff", "--name-only", "--diff-filter=AMR", args.changed_since, "HEAD") - if any(f in SELF for f in files): - files = git_files("ls-files") - else: - files = args.files - return [f for f in files if Path(f).suffix.lower() in LINTERS] - - -def escape(text, is_property=False): - text = text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") - return text.replace(":", "%3A").replace(",", "%2C") if is_property else text - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("files", nargs="*") - parser.add_argument("--changed-since", metavar="REF") - parser.add_argument("--all", action="store_true") - parser.add_argument("--print-files", action="store_true") - parser.add_argument("--github", action="store_true", help="emit GitHub annotations and a job summary") - parser.add_argument("--json", help="write findings to this file") - args = parser.parse_args() - - files = select_files(args) - if args.print_files: - print("\n".join(files)) - return - - findings = lint(files) - errors = sum(f["level"] == "ERROR" for f in findings) - total = f"{len(files)} file(s) checked, {errors} error(s), {len(findings) - errors} warning(s)" - for f in findings: - text = f"{f['where']}: {f['message']}" if f["where"] else f["message"] - if args.github: - kind = "error" if f["level"] == "ERROR" else "warning" - print(f"::{kind} file={escape(f['file'], True)},title=Workshop lint::{escape(text)}") - else: - print(f"{f['level']:5} {f['file']}: {text}") - for note in sorted(SKIPPED): - print(f"::notice title=Workshop lint::{escape(note)}" if args.github else f"NOTE {note}") - print(total) - - summary = os.environ.get("GITHUB_STEP_SUMMARY") - if args.github and summary: - lines = ["## Workshop lint", "", total, ""] - if findings: - lines += ["| | File | Where | Problem |", "|---|---|---|---|"] - for f in sorted(findings, key=lambda f: f["level"]): - problem = f["message"].replace("|", "\\|") - lines.append(f"| {'❌' if f['level'] == 'ERROR' else '⚠️'} | `{f['file']}` | {f['where']} | {problem} |") - lines += [f"- {note}" for note in sorted(SKIPPED)] - with open(summary, "a", encoding="utf-8") as out: - out.write("\n".join(lines) + "\n") - - if args.json: - Path(args.json).write_text(json.dumps(findings, indent=1)) - sys.exit(1 if errors else 0) - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/lint_workshop.yml b/.github/workflows/lint_workshop.yml index 4296aba..2f1461d 100644 --- a/.github/workflows/lint_workshop.yml +++ b/.github/workflows/lint_workshop.yml @@ -1,5 +1,5 @@ # Checks that workshop material can be opened: notebooks are valid, scripts parse. -# Runs with a read-only token and no secrets, so it is safe for pull requests from forks. +# The checks live in QLS-MiCM/MiCM_PR_Bot. Change them there, not here. name: Lint workshop material on: @@ -11,40 +11,6 @@ permissions: jobs: lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - with: - # The pull request merge commit and its first parent (the base branch) - fetch-depth: 2 - - - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install linter dependencies - run: pip install --quiet nbformat pyyaml - - - name: Select files - id: select - env: - EVENT: ${{ github.event_name }} - run: | - if [ "$EVENT" = "pull_request" ]; then - scope="--changed-since HEAD^1" - else - scope="--all" - fi - echo "scope=$scope" >> "$GITHUB_OUTPUT" - python .github/scripts/lint_workshop.py $scope --print-files | tee files.txt - if grep -qiE '\.(r|rmd|qmd)$' files.txt; then - echo "needs_r=true" >> "$GITHUB_OUTPUT" - fi - rm files.txt - - - name: Set up R - if: steps.select.outputs.needs_r == 'true' - uses: r-lib/actions/setup-r@v2 - - - name: Lint - run: python .github/scripts/lint_workshop.py ${{ steps.select.outputs.scope }} --github + uses: QLS-MiCM/MiCM_PR_Bot/.github/workflows/reusable_lint_workshop.yml@add-workshop-lint + with: + linter_ref: add-workshop-lint