diff --git a/.zenodo.json b/.zenodo.json index 17f21b4..cc6cbbd 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -30,7 +30,7 @@ ], "license": "GPL-3.0", "keywords": [ - "research data management", "simulation", "reproducibility", "containerization" + "research data management", "simulation", "reproducibility" ], "version": "1.1.2", "access_right": "open", diff --git a/README.md b/README.md index 99b3fb1..0b3bdcd 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,11 @@ Detailed descriptions of commands and APIs are provided in the dedicated interfa 5. Push project and output repositories to their remotes 6. Reuse or reference results via their output branches +The links between the project and output repositories are kept up to date by every commit made through CADET-RDM. +For commits made with plain git, an optional [pre-commit](https://pre-commit.com/) hook, `rdm-check`, does the same; see the [command line interface](https://cadet-rdm.readthedocs.io/en/latest/user_guide/command-line-interface.html) documentation. -Results are referenced by unique output branch names that encode the timestamp, active project branch, and project commit hash. CADET-RDM provides a local cache mechanism that allows results from previous runs or from other CADET-RDM projects to be reused as input data while preserving provenance information. + +Results are referenced by unique output branch names that encode the timestamp, active project branch, and project commit hash, followed by a random suffix. CADET-RDM provides a local cache mechanism that allows results from previous runs or from other CADET-RDM projects to be reused as input data while preserving provenance information. ## Getting started @@ -90,5 +93,5 @@ It includes installation instructions, usage guides for the different interfaces ## Project information - **License:** see [LICENSE](LICENSE) -- **Contributing:** see [CONTRIBUTING](CONTRIBUTING.md) -- **Authors and contributors:** see [AUTHORS](AUTHORS.md) +- **Contributing:** see [CONTRIBUTING](https://github.com/cadet/CADET-RDM/blob/main/CONTRIBUTING.md) +- **Authors and contributors:** see [AUTHORS](https://github.com/cadet/CADET-RDM/blob/main/AUTHORS.md) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1450a67..290372f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -117,7 +117,7 @@ # The theme to use for HTML and HTML Help pages. html_theme = "sphinx_book_theme" -html_logo = "_static/logo.png" +html_logo = "_static/figures/logo.png" html_theme_options = { "show_toc_level": 2, diff --git a/docs/source/index.md b/docs/source/index.md index 60de158..3d8d60a 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -11,6 +11,7 @@ user_guide/installation user_guide/getting-started user_guide/command-line-interface user_guide/python-interface +user_guide/cases user_guide/jupyter-interface ``` diff --git a/docs/source/user_guide/cases.md b/docs/source/user_guide/cases.md new file mode 100644 index 0000000..7d956cc --- /dev/null +++ b/docs/source/user_guide/cases.md @@ -0,0 +1,156 @@ +(cases)= +# Cases: options, environments and reproducible results + +CADET-RDM identifies every result by the combination of three things: + +- the **project**: the commit of the project repository that produced the result, +- the **options**: the configuration the project code was run with, +- the **environment**: the conda and pip packages installed during the run. + +A `Case` bundles a project repository with a set of `Options` and, optionally, `Environment` requirements. +It runs the project code for this combination, or reuses existing results if this combination was already run. +This makes parameter studies and comparisons reproducible without manual bookkeeping of which result belongs to which configuration. + +## Options + +`Options` is a nested dictionary that also supports attribute access. +It holds the configuration of a run and is stored as `options.json` with the results. + +```python +from cadetrdm import Options + +options = Options() +options.commit_message = "Parameter study, high flow rate" +options.debug = False +options.push = True +options.flow_rate = 2.5e-7 +options.optimizer_options = {"optimizer": "U_NSGA3", "pop_size": 16} +``` + +Values must be JSON-serializable: Python built-ins, numpy arrays and `pathlib.Path` objects are supported. +Options can be saved and loaded with `options.dump_json_file(path)` and `Options.load_json_file(path)`. + +### The options hash + +Each set of options has a hash, `options.get_hash()`, which identifies the configuration in the output log. +Not every key contributes to the hash: + +- The run control keys `commit_message`, `debug`, `push`, `force` and `branch_prefix` are excluded, since they do not change the result. +- Keys starting with an underscore (`_`) or containing a double underscore (`__`) are excluded at every nesting level. + Use them for machine-specific settings that should not affect the identity of a result, e.g. a temporary directory: + +```python +options._tmp_directory = "/dev/shm" +``` + +### Reusing options + +Because the hash depends only on the configuration, the same `Options` can be reused across projects, e.g. to run the same parameter set with different models, or copied and modified for a parameter study: + +```python +cases = [] +for flow_rate in [1.0e-7, 2.5e-7, 5.0e-7]: + case_options = options.copy() + case_options.flow_rate = flow_rate + cases.append(case_options) +``` + +## Environments + +An `Environment` describes package requirements. +Package versions are given as exact versions or as [semantic version](https://semver.org/) specifications: + +```python +from cadetrdm import Environment + +environment = Environment( + conda_packages={"cadet": ">=5.0.3"}, + pip_packages={"cadet-process": "~0.12.0", "numpy": "2.0.2"}, +) +``` + +- `"2.0.2"` requires exactly this version. +- `">=5.0.3"`, `"<6"` and similar require a version in this range. +- `"~0.12.0"` requires an approximately matching version, i.e. `0.12.x`. + +For every run, CADET-RDM records the complete environment (`conda env export` and `pip freeze`) in the `run_history` directory on the main branch of the output repository. +An `Environment` given to a `Case` is checked against these records, so only results created in a matching environment are reused. +CADET-RDM does not install packages: if the active environment does not fulfil the requirements, the case is not run. + +## Preparing the project for cases + +A `Case` imports the main package of the project repository and calls its `main` function, or the function given with `Case(..., run_method=...)`. +By default, the package is the directory named like the project repository, e.g. `my_project/my_project/__init__.py`, and a different directory can be set with `ProjectRepo(package_dir=...)`. +The function is decorated with `tracks_results`, which runs it in the tracking context and passes the project repository and the options: + +```python +# my_project/my_project/main.py +from cadetrdm import tracks_results + + +@tracks_results +def main(repo, options): + results = simulate(flow_rate=options.flow_rate) + results.save(repo.output_path / "results.h5") +``` + +```python +# my_project/my_project/__init__.py +from .main import main +``` + +The decorated function can also be called directly with options, a dictionary or the path to an options JSON file. +It requires the keys `commit_message` and `debug`, writes the options to `options.json` in the output, pushes the results if `options.push` is set, and returns the name of the new output branch together with the return value of the function. +With `options.debug = True`, the function runs without tracking the results. + +## Running and loading cases + +```python +from cadetrdm import Case, ProjectRepo + +project_repo = ProjectRepo("path/to/my_project") +case = Case(project_repo=project_repo, options=options, environment=environment) + +results_path = case.run_study() +``` + +`run_study()` + +1. updates the project and output repositories from their remotes (skipped in debug mode), +2. looks for existing results of this project commit, these options and this environment, +3. returns the path to these results if they exist, +4. otherwise runs the project code, if the active environment fulfils the environment requirements, and returns the path to the new results. + +Use `run_study(force=True)` to run the code even if results exist. +If the code fails or the environment does not match, `run_study()` returns `None`. +The execution status is stored in a `.status` file next to the project repository, so a case that is already running elsewhere on the same file system is skipped. + +To only look up existing results without running anything, use `load()`: + +```python +results_path = case.load() + +case.has_results_for_this_run +case.results_branch +``` + +`load()` does not modify the project or output repository. +It reads the output log from the local main branch of the output repository, so results that were pushed from another machine are only found after updating, e.g. with `run_study()` or `case.output_repo.update_main()`. + +By default, results must match the project commit, the options hash and the environment. +Each requirement can be relaxed, e.g. to reuse results from an earlier commit whose changes do not affect them: + +```python +results_path = case.load( + allow_commit_hash_mismatch=True, + allow_options_hash_mismatch=False, + allow_environment_mismatch=False, +) +``` + +## Results are unique and read-only + +Every run creates its own output branch, named after the time of the run, the project branch and commit, plus a random suffix (see {ref}`Python interface `). +A result therefore always points to exactly one project commit, one set of options and one recorded environment, and can be reproduced by checking out that commit and running it again with the stored `options.json`. + +Loaded results are copied into the cache directory `_cached/` of the project repository and made read-only, so a result that is used as input for further work cannot be modified by accident. diff --git a/docs/source/user_guide/command-line-interface.md b/docs/source/user_guide/command-line-interface.md index 5c19fa5..6ba9e92 100644 --- a/docs/source/user_guide/command-line-interface.md +++ b/docs/source/user_guide/command-line-interface.md @@ -106,9 +106,12 @@ rdm push --output-only Each run is stored in an output branch named: ``` -__ +[_]___ ``` +The random six-character suffix keeps runs that start at the same time apart. +The prefix is optional and set with `options.branch_prefix`. + Cache results locally: ```bash @@ -156,7 +159,7 @@ rdm remote create Example: ```bash -rdm remote create https://github.com/ githubusers_workproject Workproject githubuser +rdm remote create https://api.github.com githubusers_workproject Workproject githubuser ``` The output repository name is derived automatically by appending `_output` to the project repository name. diff --git a/docs/source/user_guide/getting-started.md b/docs/source/user_guide/getting-started.md index 1361086..4758eb1 100644 --- a/docs/source/user_guide/getting-started.md +++ b/docs/source/user_guide/getting-started.md @@ -125,6 +125,9 @@ CLI (check consistency and stage changes): rdm check ``` +Commits made through CADET-RDM keep the links between the project and output repositories up to date. +For commits made with plain git, the optional `rdm-check` pre-commit hook does the same, see [Command line interface](command-line-interface.md). + CLI (commit staged changes): ```bash @@ -137,6 +140,8 @@ CLI (push project and output repositories): rdm push ``` +To push only the output repository, e.g. to publish results without unfinished project changes, use `rdm push --output-only`. + Python (push both repositories): ```python @@ -148,9 +153,12 @@ repo.push() Results are referenced by a unique output branch name: ``` -__ +[_]___ ``` +The random six-character suffix keeps runs that start at the same time apart. +The prefix is optional and set with `options.branch_prefix`. + CLI: ```bash @@ -272,21 +280,23 @@ repo.output_repo.add_remote("git@:_output.git") CADET-RDM can create both remotes automatically if a Personal Access Token is available in the Python keyring. -The URL must match the GitHub or GitLab instance used for remote creation, for example: +Store the token under the URL of the host: `https://github.com/` for GitHub, or the URL of the GitLab instance, e.g. `https://gitlab.com/`. Store a token (Python): ```python import keyring -keyring.set_password("https://jugit.fz-juelich.de/", "username", "token") +keyring.set_password("https://github.com/", "githubuser", "token") ``` Store a token (CLI): ```bash -keyring set "https://jugit.fz-juelich.de/" +keyring set "https://github.com/" githubuser ``` +When creating the remotes, pass the API URL `https://api.github.com` for GitHub, or the URL of the GitLab instance. + Create remotes (Python): ```python @@ -296,7 +306,7 @@ repo = ProjectRepo() repo.create_remotes( name="Workproject", namespace="githubusers_workproject", - url="https://github.com/", + url="https://api.github.com", username="githubuser" ) ``` @@ -310,7 +320,7 @@ rdm remote create Example: ```bash -rdm remote create https://github.com/ githubusers_workproject Workproject githubuser +rdm remote create https://api.github.com githubusers_workproject Workproject githubuser ``` The output repository name is derived automatically by appending `_output` to the project repository name. diff --git a/docs/source/user_guide/introduction.md b/docs/source/user_guide/introduction.md index 1507f47..75d76f3 100644 --- a/docs/source/user_guide/introduction.md +++ b/docs/source/user_guide/introduction.md @@ -33,14 +33,14 @@ CADET-RDM projects are structured into two distinct repositories. :width: 700 :alt: RDM structure -CADET-RDM repository architechture +CADET-RDM repository architecture ::: Both the **project** and the **output** repository are their own git repositories. The commit architecture of CADET-RDM allows for easy tracking and reproducing of results and their respective project code. ## RDM commit architecture -Every run of the project code creates a new output branch (*result branch*) in the **output directory**. The repository on this new branch uniquely contains the files created by the execution of the project code.
At the same time, for every run of the project code the `run_history` directory on the master branch of the output repository is updated. This directory is unique to the master branch and contains the metadata and software specifications for every branch in the output repository. This directory also links the results in the output branch to the corresponding commit in the project repository used to create them. For transparency and easy accessibility, the most important specifications for every result branch are also documented in the `log.tsv` on the master branch of the output repository. +Every run of the project code creates a new output branch (*result branch*) in the **output directory**. The repository on this new branch uniquely contains the files created by the execution of the project code.
At the same time, for every run of the project code the `run_history` directory on the main branch of the output repository is updated. This directory is unique to the main branch and contains the metadata and software specifications for every branch in the output repository. This directory also links the results in the output branch to the corresponding commit in the project repository used to create them. For transparency and easy accessibility, the most important specifications for every result branch are also documented in the `log.tsv` on the main branch of the output repository. ```{eval-rst} .. subfigure:: AB @@ -55,14 +55,15 @@ Every run of the project code creates a new output branch (*result branch*) in t :alt: Output Repository :width: 420px - CADET-RDM commit architechture. + CADET-RDM commit architecture. ``` -Because of this simultanious log of the metadata and the environment used to create a specific output, results can be reproduced easily. +Because of this simultaneous log of the metadata and the environment used to create a specific output, results can be reproduced easily. ## User function The tools of CADET-RDM can be used through the command line interface (CLI), via context tracking in Python scripts or within [Jupyter Lab](https://jupyterlab.readthedocs.io/en/latest/). The following documentation contains an installation guide, a user guide to quickly start using CADET-RDM and more detailed descriptions on using the command line interface, python interface and jupyter interface. +The page on [cases](cases.md) explains how projects, options and environments together identify a result, and how results are reused instead of recomputed. diff --git a/docs/source/user_guide/python-interface.md b/docs/source/user_guide/python-interface.md index 9526fed..fcc1f03 100644 --- a/docs/source/user_guide/python-interface.md +++ b/docs/source/user_guide/python-interface.md @@ -70,6 +70,12 @@ Project and output repositories can be pushed together using a single command. repo.push() ``` +To push only the output repository: + +```python +repo.output_repo.push() +``` + Consistency checks and staging are handled automatically by the Python interface before pushing. ### Reusing results from earlier runs @@ -77,9 +83,12 @@ Consistency checks and staging are handled automatically by the Python interface Each run is stored in an output branch named: ``` -__ +[_]___ ``` +The random six-character suffix keeps runs that start at the same time apart. +The prefix is optional and set with `options.branch_prefix`. + Reuse results from a previous run by loading them into the local cache: ```python @@ -142,6 +151,7 @@ repo.output_repo.add_remote("git@:_output.git") ### Creating remotes automatically Remote repositories can be created automatically using the GitHub or GitLab APIs if a Personal Access Token is available in the Python keyring. +The token is stored under `https://github.com/` for GitHub, or the URL of the GitLab instance (see [getting started](getting-started.md)), and `url` is the API URL `https://api.github.com` for GitHub, or the URL of the GitLab instance. ```python from cadetrdm import ProjectRepo @@ -150,7 +160,7 @@ repo = ProjectRepo() repo.create_remotes( name="Workproject", namespace="githubusers_workproject", - url="https://github.com/", + url="https://api.github.com", username="githubuser" ) ``` diff --git a/examples/Apptainer.def b/examples/Apptainer.def deleted file mode 100644 index 31e0f09..0000000 --- a/examples/Apptainer.def +++ /dev/null @@ -1,28 +0,0 @@ -Bootstrap: docker -From: condaforge/miniforge3:{{ CONDA_VERSION }}-0 -Stage: build - -%arguments - CONDA_VERSION=24.11.3 - -%files - environment.yml /tmp/environment.yml - -%environment - export PYTHONDONTWRITEBYTECODE=1 - export PYTHONUNBUFFERED=1 - export DEBIAN_FRONTEND=noninteractive - -%post - apt-get update - apt-get install -y git git-lfs ssh - - apt-get clean - rm -rf /var/lib/apt/lists/* - - conda env update -n base --file /tmp/environment.yml - -%startscript - nc -lp $LISTEN_PORT - -%test diff --git a/examples/Dockerfile b/examples/Dockerfile deleted file mode 100644 index 06c470d..0000000 --- a/examples/Dockerfile +++ /dev/null @@ -1,75 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM ubuntu:noble-20250127 AS base - -ARG MINIFORGE_NAME=Miniforge3 -ARG MINIFORGE_VERSION=24.11.3-2 -ARG TARGETPLATFORM - -ENV CONDA_DIR=/opt/conda -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV PATH=${CONDA_DIR}/bin:${PATH} - -# 1. Install just enough for conda to work -# 2. Keep $HOME clean (no .wget-hsts file), since HSTS isn't useful in this context -# 3. Install miniforge from GitHub releases -# 4. Apply some cleanup tips from https://jcrist.github.io/conda-docker-tips.html -# Particularly, we remove pyc and a files. The default install has no js, we can skip that -# 5. Activate base by default when running as any *non-root* user as well -# Good security practice requires running most workloads as non-root -# This makes sure any non-root users created also have base activated -# for their interactive shells. -# 6. Activate base by default when running as root as well -# The root user is already created, so won't pick up changes to /etc/skel -RUN apt-get update > /dev/null && \ - apt-get install --no-install-recommends --yes \ - wget bzip2 ca-certificates \ - git \ - tini \ - > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/${MINIFORGE_NAME}-${MINIFORGE_VERSION}-Linux-$(uname -m).sh -O /tmp/miniforge.sh && \ - /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \ - rm /tmp/miniforge.sh && \ - conda clean --tarballs --index-cache --packages --yes && \ - find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \ - find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \ - conda clean --force-pkgs-dirs --all --yes && \ - echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ - echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc - -# Comments are provided throughout this file to help you get started. -# If you need more help, visit the Dockerfile reference guide at -# https://docs.docker.com/go/dockerfile-reference/ - -# Want to help us make this template better? Share your feedback here: https://forms.gle/ybq9Krt8jtBL3iCk7 - - -# Prevents Python from writing pyc files. -ENV PYTHONDONTWRITEBYTECODE=1 - -# Keeps Python from buffering stdout and stderr to avoid situations where -# the application crashes without emitting any logs due to buffering. -ENV PYTHONUNBUFFERED=1 - -WORKDIR /rdm_workdir - -USER root - -# Prevents interactive prompts during apt-get -ARG DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && apt-get install -y git git-lfs ssh && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -COPY environment.yml /tmp/environment.yml - -RUN conda env update -n base --file /tmp/environment.yml - -RUN pip install cadet-rdm[testing,jupyter] -RUN pip install git+https://github.com/cadet/cadet-rdm.git@master -RUN pip install git+https://github.com/cadet/cadet-rdm.git@master --force-reinstall --no-deps - -ENTRYPOINT ["tini", "--"] -CMD [ "/bin/bash" ]