Recompile generalisation lecture and add smoke tests

.github/workflows/notebook-smoke.yml

@@ -0,0 +1,78 @@
+name: Notebook smoke
+
+on:
+  pull_request:
+    paths:
+      - "_notebooks/**/*.ipynb"
+      - "scripts/smoke_notebooks.py"
+      - "scripts/smoke.yml"
+      - "requirements-smoke.txt"
+      - ".github/workflows/notebook-smoke.yml"
+  schedule:
+    # Weekly full suite (Sunday 04:00 UTC)
+    - cron: "0 4 * * 0"
+  workflow_dispatch:
+    inputs:
+      only:
+        description: "Optional notebook prefixes (space-separated), e.g. 01 12. Empty = all except skipped."
+        required: false
+        default: ""
+
+concurrency:
+  group: notebook-smoke-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: requirements-smoke.txt
+
+      - name: Install smoke dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements-smoke.txt
+
+      - name: Cache pods / notebook data downloads
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.pods_data
+            _notebooks/data
+          key: pods-data-${{ runner.os }}-${{ hashFiles('requirements-smoke.txt') }}
+          restore-keys: |
+            pods-data-${{ runner.os }}-
+
+      - name: Run notebook smoke tests
+        env:
+          MPLBACKEND: Agg
+          JUPYTER_PLATFORM_DIRS: "1"
+        run: |
+          set -euo pipefail
+          ONLY="${{ github.event.inputs.only }}"
+          if [ -n "${ONLY}" ]; then
+            # shellcheck disable=SC2086
+            python scripts/smoke_notebooks.py --only ${ONLY}
+          else
+            python scripts/smoke_notebooks.py
+          fi
+
+      - name: Upload smoke results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: notebook-smoke-results
+          path: |
+            _notebooks/test-results/summary.json
+            _notebooks/test-results/*.ok
+            _notebooks/test-results/*.fail
+          if-no-files-found: ignore

.gitignore

@@ -0,0 +1,5 @@
+.ipynb_checkpoints/
+.DS_Store
+__pycache__/
+.venv/
+_notebooks/test-results/

_notebooks/01-02-generalisation-and-neural-networks.ipynb

@@ -33,9 +33,14 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "<!-- CIP-0012 Track A: no-op outside HTML (overridden in talk-macros-html.gpp). -->\n",
+    "<!-- Class-test timer: no-op outside slides HTML (overridden in talk-macros-slides-html.gpp). -->\n",
+    "<!-- CIP-0016: mdpp expands \\diagramcredit before gpp; leftover calls are empty. -->\n",
     "<!-- Do not edit this file locally. -->\n",
     "<!-- Do not edit this file locally. -->\n",
     "<!---->\n",
+    "<!-- CIP-0012 Track A: content script load. Once-guard with alphanumeric\n",
+    "     names stays in the surrounding ifndef wrapper (GPP ifndef limitation). -->\n",
     "<!-- Do not edit this file locally. -->\n",
     "<!-- Do not edit this file locally. -->\n",
     "<!-- The last names to be defined. Should be defined entirely in terms of macros from above-->\n",
@@ -70,7 +75,9 @@
     "    plt.rcParams['text.usetex'] = False\n",
     "else:\n",
     "    plt.rcParams['text.usetex'] = True\n",
-    "    plt.rcParams['text.latex.preamble']=r'\\usepackage{amsmath}'\n",
+    "    plt.rcParams['text.latex.preamble'] = (\n",
+    "        r'\\usepackage{amsmath}\\usepackage{amssymb}\\usepackage{mathrsfs}'\n",
+    "    )\n",
     "\n",
     "plt.rcParams.update({'font.size': 22})"
    ]
@@ -127,7 +134,7 @@
     "        'mlai': 'git+https://github.com/lawrennd/mlai.git',\n",
     "        'pods': 'pods',\n",
     "        'PyDeepGP': 'git+https://github.com/SheffieldML/PyDeepGP.git',\n",
-    "        'notutils': 'notutils',\n",
+    "        'notutils': 'git+https://github.com/lawrennd/notutils.git',\n",
     "        'pods' : 'git+https://github.com/lawrennd/ods.git',\n",
     "        'lamd': 'lamd',\n",
     "        'GPy': 'gpy',\n",
@@ -143,6 +150,24 @@
     "        return f'echo \"Package {imp} is already installed.\"'"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cmd = install_command('pods')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%system {cmd}"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -1621,7 +1646,7 @@
    "source": [
     "Q, R = np.linalg.qr(Phi)\n",
     "w = sp.linalg.solve_triangular(R, Q.T@y) \n",
-    "w = pd.DataFrame(w, index=Phi.columns)\n",
+    "w = pd.DataFrame(w, index=getattr(Phi, \"columns\", None))\n",
     "w"
    ]
   },
@@ -3106,11 +3131,11 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "<img src=\"https://mlatcl.github.io/deepnn/slides/diagrams/deepnn/deep-neural-network.svg\" class=\"\" width=\"50%\" style=\"vertical-align:middle;\">\n",
+    "<img src=\"https://mlatcl.github.io/deepnn/slides/diagrams/deepnn/deep-nn1.svg\" class=\"\" width=\"50%\" style=\"vertical-align:middle;\">\n",
     "\n",
     "Figure: <i>A deep neural network</i>\n",
     "\n",
-    "<img src=\"https://mlatcl.github.io/deepnn/slides/diagrams/deepgp/deep-nn2.svg\" class=\"\" width=\"70%\" style=\"vertical-align:middle;\">\n",
+    "<img src=\"https://mlatcl.github.io/deepnn/slides/diagrams/deepnn/deep-nn2.svg\" class=\"\" width=\"70%\" style=\"vertical-align:middle;\">\n",
     "\n",
     "Figure: <i>A deep neural network. Input nodes are shown at the bottom.\n",
     "Each hidden layer is the result of applying an affine transformation to\n",
@@ -3367,8 +3392,6 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "::: {.cell .markdown}\n",
-    "\n",
     "## References\n",
     "\n",
     "bootstrap\n",

requirements-smoke.txt

@@ -0,0 +1,28 @@
+# Runtime + execution harness for lecture notebook smoke tests.
+# Install: pip install -r requirements-smoke.txt
+
+# Notebook execution
+nbclient>=0.10,<1
+nbformat>=5.10,<6
+ipykernel>=6.29
+ipywidgets>=8.1
+ipython>=8.0
+
+# Teaching stack (match lecture %pip install cells)
+GPy>=1.13
+git+https://github.com/lawrennd/mlai.git#egg=mlai
+git+https://github.com/lawrennd/ods.git#egg=pods
+git+https://github.com/lawrennd/notutils.git#egg=notutils
+
+# Used by lecture notebooks
+matplotlib>=3.8
+numpy>=1.26
+scipy>=1.11
+pandas>=2.0
+scikit-learn>=1.3
+scikit-image>=0.22
+Pillow>=10.0
+PyYAML>=6.0
+plotly>=5.0
+requests>=2.31
+daft-pgm>=0.1

scripts/smoke.yml

@@ -0,0 +1,14 @@
+# Course-specific notebook smoke defaults (scripts/smoke_notebooks.py).
+# skip: filename prefixes to omit. plot_dirs: extra dirs created under _notebooks.
+# timeout_by_prefix: optional per-prefix cell timeouts in seconds.
+skip:
+  - 01-02-tmp          # leftover compile artefact
+  - 01-02-01-background  # orphan split; not in lectures.csv
+  - 01-02-02-background  # orphan split; not in lectures.csv
+  - 01-02-03-background  # orphan split; not in lectures.csv
+  - 02-01-automatic-differentiation  # needs jax, not in the shared smoke env
+  - 03-02-assignment   # student assignment; needs torch
+  - 05-02-assignment   # student assignment; needs torch
+plot_dirs: []
+timeout: 900
+timeout_by_prefix: {}