WIP: Some tweaks in tutorial notebooks

.github/workflows/docs-stable.yml

@@ -24,7 +24,7 @@ jobs:
 
       - name: Publish docs
         env:
-          # Pin "Open in Colab" links to the commit these docs are built from	
+          # Pin "Open in molab" links to the commit these docs are built from	
           DOCS_GIT_REF: ${{ github.sha }}
         run: |
           git config user.name github-actions

.github/workflows/notebook_testing.yml

@@ -22,6 +22,7 @@ jobs:
         with:
           files: |
             .github/workflows/notebook_testing.yml
+            docs/tutorial/**
             src/**
             tests/**
             pyproject.toml
@@ -33,5 +34,29 @@ jobs:
       - name: Run tests
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
-          pytest --nbmake docs/tutorial/*ipynb --no-cov
-          pytest --nbmake docs/tutorial/advanced/*ipynb --no-cov
+          marimo check docs/tutorial/basic/*.py docs/tutorial/advanced/*.py
+
+      # The documentation renders the committed `.ipynb` exports of the notebooks without executing them,
+      # so they have to be re-exported whenever a notebook changes.
+      - name: Check exported notebooks are in sync
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          python docs/export_tutorials.py --check
+
+      # marimo notebooks are Python scripts; running them executes all cells
+      # and exits with a non-zero status if any cell raises an error.
+      - name: Run tutorial notebooks
+        if: steps.changed-files.outputs.any_changed == 'true'
+        env:
+          MPLBACKEND: Agg
+        run: |
+          failed=()
+          for notebook in docs/tutorial/basic/*.py docs/tutorial/advanced/*.py; do
+            echo "::group::${notebook}"
+            python "${notebook}" || failed+=("${notebook}")
+            echo "::endgroup::"
+          done
+          if [ ${#failed[@]} -gt 0 ]; then
+            printf '::error::Notebook failed: %s\n' "${failed[@]}"
+            exit 1
+          fi

.github/workflows/release.yml

@@ -0,0 +1,48 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+
+  release:
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.3.2
+        with:
+          python-version: "3.13"
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      - name: Publish to PyPI
+        run: uv publish --trusted-publishing always
+
+      - name: Create release tag
+        # Adding a `--prerelease` flag in `gh release create` makes the release not
+        # show up as the latest stable release.
+        run: |
+          version="$(uv version --short)"
+          if uv run --no-project --with packaging python -c \
+              "import sys; from packaging.version import Version; sys.exit(0 if Version('${version}').is_prerelease else 1)"; then
+            prerelease="--prerelease"
+          else
+            prerelease=""
+          fi
+          gh release create "v${version}" --title "v${version}" --generate-notes ${prerelease}
+        env:
+          GH_TOKEN: ${{ github.token }}

.gitignore

@@ -80,6 +80,9 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 
+# Marimo
+__marimo__
+
 # IPython
 profile_default/
 ipython_config.py

README.md

@@ -31,7 +31,6 @@ Online documentation is available at [pathpy.net](https://www.pathpy.net).
 
 The documentation includes multiple tutorials that introduce the use of pathpyG to model temporal graph and path data. You will also find an API reference and other useful information that will help you to get started.
 
-
 Dependencies
 ------------
 
@@ -62,3 +61,43 @@ Development
 pathpyG development takes place on Github: https://github.com/pathpy/pathpyG
 
 Please submit any reproducible bugs you encounter to the [issue tracker](https://github.com/pathpy/pathpyG/issues).
+
+To build the notebooks and the documentation locally, follow these steps:
+
+1. Install the package with `pip install -e .[vis,cpu] --group doc`.
+
+Unless using `uv`, you may have to specify extra URLs for `pip` to install `pytorch` successfully:
+
+```bash
+pip install -e ".[vis,cpu]" --group doc \
+  --extra-index-url https://download.pytorch.org/whl/cpu \
+  --find-links https://data.pyg.org/whl/torch-2.8.0+cpu.html
+```
+
+2. Run `python docs/export_tutorials.py` to generate the tutorials.
+
+```bash
+python docs/export_tutorials.py docs/tutorial/basic/*.py docs/tutorial/advanced/*.py
+python docs/export_tutorials.py --check
+```
+
+3. Create a local branch `docs-local` and generate the `dev` documentation on that branch.
+
+```bash
+git branch -D docs-local
+VERSION=$(python -c "import pathpyG; print(pathpyG.__version__)")
+mike deploy -b docs-local --update-aliases "$VERSION-dev" dev
+```
+
+4. Generate the `stable` documentation on the `docs-local` branch.
+
+```bash
+DOCS_GIT_REF=$(git rev-parse HEAD) mike deploy -b docs-local --update-aliases "$VERSION" stable
+mike set-default -b docs-local stable
+```
+
+5. Serve the documentation from the `docs-local` branch.
+
+```bash
+mike serve -b docs-local
+```

docs/contributing.md

@@ -32,7 +32,7 @@ To ensure version consistency, we use a [Development Container](https://containe
 
 ## Documentation
 
-This project uses [`MkDocs`](https://www.mkdocs.org/) for documentation. It is a static site generator that creates the necessary `html`-files automatically from the `markdown`-files and [:jupyter_logo: Jupyter](https://jupyter.org/) notebooks in the `docs/`-directory and the `Python`-files in `src/`. The documentation is hosted on GitHub Pages.
+This project uses [`MkDocs`](https://www.mkdocs.org/) for documentation. It is a static site generator that creates the necessary `html`-files automatically from the `markdown`-files and [marimo](https://marimo.io/) notebooks in the `docs/`-directory and the `Python`-files in `src/`. The documentation is hosted on GitHub Pages.
 
 ### Hosting the documentation locally
 
@@ -85,7 +85,23 @@ If you want to ignore specific `.py`-files in the code reference, you can add th
 
 ### Tutorials
 
-The tutorials are written in :jupyter_logo: Jupyter notebooks. They are located in the `docs/`-directory. You can add new tutorials by adding the notebook to the `docs/tutorial/`-directory and adding the path to the `mkdocs.yml`-file under `nav:`. The tutorials are automatically converted to `html`-files when the documentation is built.
+The tutorials are written as [marimo](https://marimo.io/) notebooks, i.e. plain `.py`-files, and are located in the `docs/tutorial/basic/`- and `docs/tutorial/advanced/`-directories. You can open and edit a tutorial with
+```bash
+marimo edit docs/tutorial/basic/basic_concepts.py
+```
+
+Executing all tutorials takes a long time, so the documentation does not execute them when it is built. Instead, each notebook is exported (including the outputs of all cells) to a Jupyter notebook next to it, which is committed to the repository and rendered by `mkdocs-jupyter`. After changing a tutorial, re-export it with
+```bash
+uv run --extra cpu --extra vis python docs/export_tutorials.py docs/tutorial/basic/basic_concepts.py
+```
+Without any arguments, `docs/export_tutorials.py` re-exports all notebooks whose `.ipynb` is out of sync with their `.py`. CI checks that all exported notebooks are in sync.
+
+You can add new tutorials by adding the notebook to one of the tutorial directories, exporting it as described above and adding the path to the `mkdocs.yml`-file under `nav:`. Note that the path in `nav:` needs to end with `.ipynb` instead of `.py`.
+
+All tutorials are tested in CI by running them as Python scripts. You can do the same locally with
+```bash
+python docs/tutorial/basic/basic_concepts.py
+```
 
 ### Adding new pages
 

docs/export_tutorials.py

@@ -0,0 +1,108 @@
+"""Export the marimo tutorials in `docs/tutorial/` to the Jupyter notebooks rendered in the documentation.
+
+The marimo notebooks (`*.py`) are the source of the tutorials. Since executing them takes a long time,
+the exported Jupyter notebooks (`*.ipynb`), including the outputs of all cells, are committed to the
+repository and rendered by the `mkdocs-jupyter` plugin without executing them again.
+
+Usage:
+
+    # Re-export (and execute) all notebooks whose `.ipynb` is out of sync with its `.py`
+    uv run --extra cpu python docs/export_tutorials.py
+
+    # Re-export specific notebooks, e.g. after a change to pathpyG that affects their outputs
+    uv run --extra cpu python docs/export_tutorials.py docs/tutorial/basic/basic_concepts.py
+
+    # Only check that all `.ipynb` are in sync with their `.py` (exits with status 1 otherwise)
+    uv run python docs/export_tutorials.py --check
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+TUTORIAL_DIR = Path("docs", "tutorial")
+MAX_WORKERS = 4
+# Share the CPU cores between the notebooks exported in parallel. Otherwise, each notebook uses all cores for torch
+# operations, and the resulting oversubscription slows down the export drastically
+THREADS_PER_WORKER = str(max(1, (os.cpu_count() or 1) // MAX_WORKERS))
+
+
+def export(notebook: Path, output: Path, execute: bool) -> None:
+    """Export a marimo notebook to a Jupyter notebook, optionally including the outputs of all cells."""
+    cmd = [sys.executable, "-m", "marimo", "export", "ipynb", str(notebook), "-o", str(output), "--sort", "top-down"]
+    if execute:
+        cmd.append("--include-outputs")
+    env = {"OMP_NUM_THREADS": THREADS_PER_WORKER, "MKL_NUM_THREADS": THREADS_PER_WORKER} | os.environ
+    result = subprocess.run(cmd, capture_output=True, text=True, env=env)
+    if result.returncode != 0:
+        raise RuntimeError(f"Exporting {notebook} failed:\n{result.stdout}\n{result.stderr}")
+
+
+def cell_sources(ipynb: Path) -> list[tuple[str, str]]:
+    """Return the type and source of all cells of a Jupyter notebook, ignoring their outputs."""
+    cells = json.loads(ipynb.read_text("utf-8"))["cells"]
+    return [(cell["cell_type"], "".join(cell["source"])) for cell in cells]
+
+
+def is_in_sync(notebook: Path) -> bool:
+    """Check whether the committed `.ipynb` of a marimo notebook contains the same cells as the notebook itself."""
+    ipynb = notebook.with_suffix(".ipynb")
+    if not ipynb.exists():
+        return False
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        exported = Path(tmp_dir, ipynb.name)
+        export(notebook, exported, execute=False)
+        return cell_sources(exported) == cell_sources(ipynb)
+
+
+def update(notebook: Path) -> None:
+    """Execute a marimo notebook and export it, including all outputs, to the `.ipynb` next to it."""
+    print(f"Exporting {notebook}", flush=True)
+    export(notebook, notebook.with_suffix(".ipynb"), execute=True)
+    print(f"Finished exporting {notebook}", flush=True)
+
+
+def main() -> int:
+    """Re-export or check the tutorial notebooks and return the exit status."""
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        "notebooks", nargs="*", type=Path, help="marimo notebooks to re-export (default: all out of sync)"
+    )
+    parser.add_argument("--check", action="store_true", help="only check that all notebooks are in sync")
+    args = parser.parse_args()
+
+    # Notebooks prefixed with `_` (e.g. in `archive/`) are not part of the documentation
+    notebooks = args.notebooks or [
+        nb for nb in sorted(TUTORIAL_DIR.rglob("*.py")) if not any(p.startswith("_") for p in nb.parts)
+    ]
+    # Explicitly given notebooks are always re-exported, all others only if they are out of sync
+    if args.check or not args.notebooks:
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            notebooks = [nb for nb, ok in zip(notebooks, executor.map(is_in_sync, notebooks)) if not ok]
+    if args.check:
+        for notebook in notebooks:
+            print(f"{notebook.with_suffix('.ipynb')} is out of sync with {notebook}")
+        if notebooks:
+            print("Run `uv run --extra cpu python docs/export_tutorials.py` to re-export them")
+        return 1 if notebooks else 0
+
+    failed = []
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        for notebook, future in [(nb, executor.submit(update, nb)) for nb in notebooks]:
+            try:
+                future.result()
+            except RuntimeError as e:
+                print(e, file=sys.stderr)
+                failed.append(notebook)
+    if failed:
+        print(f"Failed to export: {', '.join(map(str, failed))}", file=sys.stderr)
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docs/overrides/assets/stylesheets/header.css

@@ -1,4 +1,7 @@
-.md-header__topic {
+/* Hide the site name (the logo already contains it) and the page title that replaces it on scroll.
+   The first topic itself has to stay visible, since mike's version selector is appended to it. */
+.md-header__topic:first-child > .md-ellipsis,
+.md-header__topic[data-md-component="header-topic"] {
   display: none;
 }
 
@@ -9,4 +12,4 @@
 
 .md-header__button.md-logo img, .md-header__button.md-logo svg {
   height: 2.2rem;
-}
+}

docs/overrides/main.html

@@ -7,14 +7,15 @@
   {%- endif %}
 {%- endblock %}
 
-<!-- Includes a download button in the Jupyter tutorials -->
+<!-- Includes download and "Open in molab" buttons in the marimo tutorials -->
 {% block content %}
 {% if page.nb_url %}
-    <a href="{{ page.nb_url }}" title="Download Notebook" class="md-content__button md-icon">
+    {% set nb_src = page.file.src_uri | replace(".ipynb", ".py") %}
+    <a href="{{ base_url }}/{{ nb_src }}" title="Download Notebook" class="md-content__button md-icon" download>
         {% include ".icons/material/download.svg" %}
     </a>
-    <a href="https://colab.research.google.com/github/pathpy/pathpyG/blob/{{ config.extra.git_ref }}/docs/{{ page.file.src_uri }}" title="Open in Google Colab" class="md-content__button md-icon">
-      {% include ".icons/simple/googlecolab.svg" %}
+    <a href="https://molab.marimo.io/github/pathpy/pathpyG/blob/{{ config.extra.git_ref }}/docs/{{ nb_src }}" title="Open in molab" class="md-content__button md-icon">
+      {% include ".icons/material/rocket-launch.svg" %}
     </a>
 {% endif %}
 {{ super() }}

docs/reference/pathpyG/visualisations/index.md

@@ -516,4 +516,4 @@ You can find the default values for each argument in `pathpyG.toml` located in t
     However, if you want to change either `color`, `size`, or `opacity` for nodes or edges, the preferred way is to use the dedicated keyword arguments described in the previous sections.
 
 ---
-For more details and usage examples, see [Manim Visualisation Tutorial](/tutorial/manim_tutorial),[Visualisation Tutorial](/tutorial/visualisation) and [Develop your own plot Functions](/plot_tutorial)
+For more details and usage examples, see [Manim Visualisation Tutorial](/tutorial/basic/manim_tutorial),[Visualisation Tutorial](/tutorial/basic/visualisation) and [Develop your own plot Functions](/plot_tutorial)