diff --git a/README.md b/README.md index 7879fef..9fe94d1 100644 --- a/README.md +++ b/README.md @@ -35,17 +35,49 @@ Call `path.done()` to check progress without blocking, or `path.result()` to wai explicitly. Pass `download(background=False)` to block and get the path as a plain string instead. -The download location defaults to `~/em_database` and can be changed either for the -session or through the `EM_DATABASE_DATA_DIR` environment variable: +## Settings + +Configuration lives in a live, matplotlib-`rcParams`-style object, +`em_database.settings`, seeded at import from `~/.em_database/settings.yaml`. +Change it in memory for an immediate effect, and persist it to remember the +choice across sessions: ```python import em_database -em_database.set_data_dir("/path/to/somewhere") -em_database.get_data_dir() -em_database.reset_data_dir() +em_database.settings["data_dir"] = "/big/disk/em_data" # takes effect now +em_database.settings.save() # remember it next time ``` +In Jupyter you can also edit them interactively — `display(em_database.settings)` +renders a panel to set (and save) the data directory directly. + +The data directory defaults to `~/em_database`. Convenience helpers wrap the +common case — `set_data_dir` persists by default: + +```python +em_database.get_data_dir() # current location +em_database.set_data_dir("/big/disk/em_data") # set + persist +em_database.set_data_dir("/scratch", persist=False) # one-off, in-memory only +em_database.reset_data_dir() # back to the default, forget the choice +``` + +No environment variable is needed. Only values you explicitly set are written to +the file, so the default is never frozen in — it keeps being obeyed even if it +changes. For a one-off override (e.g. CI) the legacy `EM_DATABASE_DATA_DIR` is +still honored, and `EM_DATABASE_CONFIG` relocates the settings file. + +### Shared, system-wide data + +Datasets can be installed once for every user. `download()` and `filepath()` +look in the shared/system locations **first**, then your own data directory, and +only download (into your directory) if the file is nowhere to be found — so a +shared copy is reused instead of refetched. Shared locations come from the +`EM_DATABASE_SHARED_DIR` environment variable (an `os.pathsep`-separated list), a +`shared_data_dirs` list in your settings, or a system config file +(`/etc/em_database/settings.yaml`, or `%PROGRAMDATA%\em_database\settings.yaml` +on Windows). + ## Adding a dataset Datasets are described by a YAML file in `em_database/datasets/`, one entry per file, diff --git a/em_database/__init__.py b/em_database/__init__.py index c9f9cc2..adacee5 100644 --- a/em_database/__init__.py +++ b/em_database/__init__.py @@ -1,19 +1,12 @@ ### Example datasets ### import os +from em_database.config import settings from em_database.downloadable_dataset import DownloadableDataset from em_database._create_stubs import build_docstring from em_database import data __all__ = [] -if "EM_DATABASE_DATA_DIR" not in os.environ: - # set the default dir to User's home directory + "/emdata" - os.environ["EM_DATABASE_DATA_DIR"] = os.path.join( - os.path.expanduser("~"),"em_database" - ) - - - def get_data_dir(): """ Get the directory where example datasets are stored. @@ -23,9 +16,10 @@ def get_data_dir(): str Path to the example datasets directory. """ - return os.environ["EM_DATABASE_DATA_DIR"] + from em_database import config + return config.data_dir() -def set_data_dir(path: str): +def set_data_dir(path: str, persist: bool = True): """ Set the directory where example datasets are stored. @@ -33,16 +27,31 @@ def set_data_dir(path: str): ---------- path : str Path to the desired example datasets directory. + persist : bool, optional + If True (the default), remember the choice across sessions by writing it + to the settings file. Pass False for a one-off, in-memory change. """ - os.environ["EM_DATABASE_DATA_DIR"] = path + settings["data_dir"] = str(path) + if persist: + settings.save() def reset_data_dir(): """ - Reset the example datasets directory to the default location. + Reset the example datasets directory to the default location, clearing any + saved choice. """ - os.environ["EM_DATABASE_DATA_DIR"] = os.path.join( - os.path.expanduser("~"),"em_database" - ) + settings.reset("data_dir") + + +def get_setting(key: str, default=None): + """Read a value from :data:`em_database.settings`.""" + return settings.get(key, default) + +def set_setting(key: str, value, persist: bool = True): + """Set a value in :data:`em_database.settings`, persisting it by default.""" + settings[key] = value + if persist: + settings.save() def browse(**kwargs): @@ -60,7 +69,8 @@ def browse(**kwargs): return _browse(**kwargs) -__all__ = ['get_data_dir', 'set_data_dir', 'reset_data_dir', 'browse', "data"] +__all__ = ['get_data_dir', 'set_data_dir', 'reset_data_dir', + 'get_setting', 'set_setting', 'settings', 'browse', "data"] # Let ``display(em_database)`` render the browser. Reassigning the module's diff --git a/em_database/config.py b/em_database/config.py new file mode 100644 index 0000000..1fa7bb3 --- /dev/null +++ b/em_database/config.py @@ -0,0 +1,213 @@ +"""Runtime settings for em_database, in the style of matplotlib's ``rcParams``. + +``em_database.settings`` is a live dict, seeded at import from a YAML file (or +the built-in defaults). Change it in memory:: + + import em_database + em_database.settings["data_dir"] = "/big/disk/em_data" + +and it takes effect immediately. To remember the choice across sessions, persist +it:: + + em_database.settings.save() + +Settings live in a ``~/.em_database`` folder (``~/.em_database/settings.yaml``), +or wherever ``EM_DATABASE_CONFIG`` points. No environment variable is *needed* to +configure em_database - the settings object is the source of truth; the old +``EM_DATABASE_DATA_DIR`` is still honored as a seed for backward compatibility. + +Data can also be installed **system-wide** and shared by every user. When +resolving where a dataset lives, em_database looks in the shared/system +locations first, then the user's own data directory, and only downloads (into +the user directory) if it is nowhere to be found. Shared locations come from a +system config file (see :func:`system_config_path`), the ``EM_DATABASE_SHARED_DIR`` +environment variable (an ``os.pathsep``-separated list), or a ``shared_data_dirs`` +list in the user's settings. +""" +from __future__ import annotations + +import os +from pathlib import Path + +import yaml + + +def _default_data_dir() -> str: + return os.path.join(os.path.expanduser("~"), "em_database") + + +#: Built-in defaults - the fallback when nothing is configured. +DEFAULTS: dict = { + "data_dir": _default_data_dir(), +} + + +def config_dir() -> Path: + """The ``~/.em_database`` folder that holds the settings file.""" + return Path(os.path.expanduser("~")) / ".em_database" + + +def config_path() -> Path: + """Path to the YAML settings file. + + ``EM_DATABASE_CONFIG`` overrides the location; otherwise it is + ``~/.em_database/settings.yaml``. + """ + override = os.environ.get("EM_DATABASE_CONFIG") + if override: + return Path(override) + return config_dir() / "settings.yaml" + + +def system_config_path() -> Path | None: + """Path to the machine-wide settings file (for system-wide installs). + + ``EM_DATABASE_SYSTEM_CONFIG`` overrides it; otherwise it is + ``/etc/em_database/settings.yaml`` (POSIX) or + ``%PROGRAMDATA%\\em_database\\settings.yaml`` (Windows). + """ + override = os.environ.get("EM_DATABASE_SYSTEM_CONFIG") + if override: + return Path(override) + if os.name == "nt": + base = os.environ.get("PROGRAMDATA") + return Path(base) / "em_database" / "settings.yaml" if base else None + return Path("/etc/em_database/settings.yaml") + + +def _read_yaml(path: Path | None) -> dict: + """Parse a YAML file into a dict, or ``{}`` if missing/unreadable/malformed.""" + if path is None: + return {} + try: + with open(path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) + return data if isinstance(data, dict) else {} + except FileNotFoundError: + return {} + except Exception: + # A malformed settings file must not break importing the package. + return {} + + +def _read_file() -> dict: + """The user settings stored on disk, or ``{}``.""" + return _read_yaml(config_path()) + + +def _read_system() -> dict: + """The machine-wide settings, or ``{}``.""" + return _read_yaml(system_config_path()) + + +def _write_file(data: dict) -> None: + path = config_path() + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + yaml.safe_dump(dict(data), handle, default_flow_style=False, sort_keys=True) + + +def _seed(settings: "Settings") -> None: + """(Re)seed a settings object: defaults < file < legacy env var.""" + settings.clear() + settings.update(DEFAULTS) + settings.update(_read_file()) # the file overrides the defaults + # Backward compatibility: seed from the old env var if it is set. + env_dir = os.environ.get("EM_DATABASE_DATA_DIR") + if env_dir: + settings["data_dir"] = env_dir + + +class Settings(dict): + """A live settings mapping that can persist itself to the YAML file.""" + + def save(self) -> None: + """Persist the settings to the YAML file. + + Only values that DIFFER from the built-in defaults are written, so a + default (like the data directory) is never frozen into the file: it stays + dynamic and keeps being obeyed across sessions, machines, or a change to + the default itself. When nothing differs, any stale file is removed. + """ + delta = {key: value for key, value in self.items() if DEFAULTS.get(key) != value} + path = config_path() + if delta: + _write_file(delta) + elif path.exists(): + path.unlink() + + def reload(self) -> None: + """Re-seed from the built-in defaults and the on-disk file, discarding + any in-memory changes.""" + _seed(self) + + def reset(self, key: str | None = None) -> None: + """Reset a key (or everything) to its default, in memory and on disk.""" + if key is None: + self.clear() + self.update(DEFAULTS) + elif key in DEFAULTS: + self[key] = DEFAULTS[key] + else: + self.pop(key, None) + self.save() # delta-save drops the now-default key from the file + + def widget(self): + """Return an interactive widget for editing the settings (Jupyter).""" + from em_database.widget import settings_widget + return settings_widget() + + def _repr_mimebundle_(self, **kwargs): + """Render as an editable settings widget in Jupyter (falls back to the + plain dict repr if anywidget is not installed).""" + try: + from em_database.widget import settings_widget + widget = settings_widget() + except Exception: + return {"text/plain": repr(dict(self))} + return widget._repr_mimebundle_(**kwargs) + + +#: The live settings object (seeded once at import). +settings = Settings() +_seed(settings) + + +def data_dir() -> str: + """The user's data directory - where downloads are written.""" + return str(settings.get("data_dir") or _default_data_dir()) + + +def shared_data_dirs() -> list[str]: + """System-wide / shared data locations, checked before the user's dir. + + Order: ``EM_DATABASE_SHARED_DIR`` (an ``os.pathsep`` list), then the user's + ``shared_data_dirs`` setting, then the system config file's ``data_dir`` and + ``shared_data_dirs``. Duplicates are removed, order preserved. + """ + dirs: list[str] = [] + env = os.environ.get("EM_DATABASE_SHARED_DIR") + if env: + dirs += [d for d in env.split(os.pathsep) if d] + dirs += [str(d) for d in (settings.get("shared_data_dirs") or [])] + system = _read_system() + if system.get("data_dir"): + dirs.append(str(system["data_dir"])) + dirs += [str(d) for d in (system.get("shared_data_dirs") or [])] + seen: set[str] = set() + unique: list[str] = [] + for d in dirs: + if d not in seen: + seen.add(d) + unique.append(d) + return unique + + +def data_search_dirs() -> list[str]: + """Everywhere to look for an existing dataset: shared/system dirs first, + then the user's data directory.""" + dirs = shared_data_dirs() + user = data_dir() + if user not in dirs: + dirs.append(user) + return dirs diff --git a/em_database/downloadable_dataset.py b/em_database/downloadable_dataset.py index e07c9f0..ad7cfbe 100644 --- a/em_database/downloadable_dataset.py +++ b/em_database/downloadable_dataset.py @@ -129,8 +129,8 @@ def _repr_mimebundle_(self, **kwargs): def _resolve_destination(destination: str | None) -> str: """Return the directory the dataset should live in.""" if destination is None: - return os.environ.get("EM_DATABASE_DATA_DIR", - os.path.join(os.path.expanduser("~"), "em_database")) + from em_database import config + return config.data_dir() return destination def download(self, @@ -176,7 +176,12 @@ def download(self, """ if not background: return self._retrieve(destination, progressbar, chunk_size) - target = os.path.join(self._resolve_destination(destination), self.file) + # Resolve where the file will end up: an existing shared/user copy if it + # is already present, otherwise the user's download location. + if destination is not None: + target = os.path.join(destination, self.file) + else: + target = self.filepath() or os.path.join(self._resolve_destination(None), self.file) # In Jupyter (with the widget installed) a background download pops a # cancelable toast; the toast's monitor replaces the plain progress bar. monitor = finish = None @@ -198,14 +203,25 @@ def _retrieve(self, destination: str | None = None, progressbar: bool = True, chunk_size: int = 4096) -> str: - """Fetch the file with pooch and return its local path (blocking).""" + """Fetch the file and return its local path (blocking). + + With no explicit destination, an existing system-wide/shared copy is used + as-is (never re-downloaded); otherwise pooch downloads into the user's + data directory. + """ if progressbar: try: import tqdm # noqa: F401 except ImportError: print("`tqdm` is not installed, progress bar will be disabled.") progressbar = False - destination = self._resolve_destination(destination) + if destination is None: + shared = self._find_shared() + if shared is not None: + return shared + destination = self._resolve_destination(None) + else: + destination = self._resolve_destination(destination) # Instantiate an Http downloader with a custom user agent headers = {"User-Agent": "em_database (https://github.com/CSSFrancis/em_data)"} downloader = pooch.HTTPDownloader(progressbar=progressbar, @@ -220,17 +236,26 @@ def _retrieve(self, ) return filepath + def _find_shared(self) -> str | None: + """Path to an existing copy in a shared/system data dir, or None.""" + from em_database import config + for directory in config.shared_data_dirs(): + candidate = os.path.join(directory, self.file) + if os.path.exists(candidate): + return candidate + return None + def filepath(self) -> str: - """ Return the local file path of the dataset if downloaded. - - If not downloaded return None. """ - destination = os.environ.get("EM_DATABASE_DATA_DIR", - os.path.join(os.path.expanduser("~"), "em_database")) - filepath = os.path.join(destination, self.file) - if os.path.exists(filepath): - return filepath - else: - return None + """ Return the local file path of the dataset if present. + + Looks in the shared/system data locations first, then the user's data + directory. Returns None if the dataset is not downloaded anywhere. """ + from em_database import config + for directory in config.data_search_dirs(): + candidate = os.path.join(directory, self.file) + if os.path.exists(candidate): + return candidate + return None def delete(self, destination: str | None = None) -> bool: """ Delete the downloaded file if it is present. diff --git a/em_database/static/browser.css b/em_database/static/browser.css index b4eb66d..3ba9959 100644 --- a/em_database/static/browser.css +++ b/em_database/static/browser.css @@ -196,6 +196,39 @@ .emdb-inline-toast.cancelling { opacity: 0.65; } .emdb-inline-toast.error { border-color: var(--emdb-red); } +/* Settings panel (display(em_database.settings)) ------------------- */ +.emdb-settings-host { max-width: 620px; } +.emdb-settings { padding: 16px 18px; } +.emdb-field { margin-top: 12px; } +.emdb-field-label { + font-size: 10.5px; font-weight: 700; letter-spacing: 0.06em; + text-transform: uppercase; color: var(--emdb-muted); margin: 12px 0 6px; +} +.emdb-field-row { display: flex; gap: 8px; align-items: stretch; } +.emdb-field-row .emdb-search { flex: 1; min-width: 0; margin-top: 0; } +.emdb-browse { + flex: 0 0 auto; cursor: pointer; font-family: var(--emdb-font); + font-size: 12px; font-weight: 600; color: var(--emdb-text); + background: var(--emdb-surface0); border: 1px solid var(--emdb-surface1); + border-radius: 7px; padding: 0 14px; +} +.emdb-browse:hover { border-color: var(--emdb-blue); color: var(--emdb-blue); } +.emdb-field-actions { display: flex; gap: 8px; margin-top: 8px; } +.emdb-hint { margin-top: 6px; font-size: 11px; color: var(--emdb-subtext); } +.emdb-status { margin-top: 12px; font-size: 12px; color: var(--emdb-green); font-weight: 600; } +.emdb-settings-info { margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--emdb-surface0); } +.emdb-searchlist { display: flex; flex-direction: column; gap: 5px; } +.emdb-searchdir { + font-family: var(--emdb-mono); font-size: 11.5px; color: var(--emdb-text); + word-break: break-all; +} +.emdb-tag { + font-family: var(--emdb-font); font-size: 10px; font-weight: 600; + color: var(--emdb-crust); background: var(--emdb-green); + border-radius: 4px; padding: 1px 6px; white-space: nowrap; +} +.emdb-tag.shared { background: var(--emdb-blue); } + /* Toasts: pinned to the bottom-right of the viewport ----------------- */ .emdb-toast-root { position: fixed; right: 16px; bottom: 16px; z-index: 99999; diff --git a/em_database/static/settings.js b/em_database/static/settings.js new file mode 100644 index 0000000..41bacd0 --- /dev/null +++ b/em_database/static/settings.js @@ -0,0 +1,134 @@ +// settings.js - an editable settings panel (what display(em_database.settings) shows). +// +// Edit the data directory directly: type a path and Save (persist), Use for +// session (in-memory), or Reset to the default. Also shows the settings-file +// location and the shared-then-user search order. + +function esc(value) { + return String(value).replace(/[&<>"']/g, function (c) { + return { "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]; + }); +} + +function el(tag, cls, html) { + const n = document.createElement(tag); + if (cls) n.className = cls; + if (html != null) n.innerHTML = html; + return n; +} + +function render({ model, el: root }) { + root.classList.add("emdb", "emdb-settings-host"); + + let nonce = 0; + function cmd(action, extra) { + model.set("_command", Object.assign({ action, nonce: nonce++ }, extra || {})); + model.save_changes(); + } + + function draw() { + root.innerHTML = ""; + const wrap = el("div", "emdb-settings"); + wrap.appendChild(el("div", "emdb-brand", '◆ Settings')); + + // Data directory field + const field = el("div", "emdb-field"); + field.appendChild(el("div", "emdb-field-label", "Data directory")); + // Input + Browse on one line (Browse to the right of the path field). + const row = el("div", "emdb-field-row"); + const input = el("input", "emdb-search"); + input.type = "text"; + input.value = model.get("data_dir") || ""; + input.placeholder = model.get("default_dir") || ""; + input.spellcheck = false; + row.appendChild(input); + const browse = el("button", "emdb-browse", "Browse…"); + browse.title = "Pick any file inside the folder you want; its folder is used"; + row.appendChild(browse); + field.appendChild(row); + + // Client-side picker: a plain file (a normal "Open" dialog, no + // "upload" prompt). We use the selected file's parent as the directory. The + // absolute path is available on desktop/Electron (File.path); a plain + // browser hides it, so we fall back to asking the user to type it. + const picker = el("input"); + picker.type = "file"; + picker.style.display = "none"; + field.appendChild(picker); + const hint = el("div", "emdb-hint"); + field.appendChild(hint); + + browse.addEventListener("click", function () { picker.click(); }); + picker.addEventListener("change", function () { + const f = picker.files && picker.files[0]; + if (f) { + if (f.path) { // absolute path available (desktop/Electron) + const dir = f.path.slice(0, f.path.length - f.name.length).replace(/[\\/]+$/, ""); + input.value = dir; + hint.textContent = "Selected " + dir + " — click Save to keep it."; + } else { + hint.textContent = + "This browser can’t share the folder’s path — type it above instead."; + } + } + picker.value = ""; // let the same file be picked again + }); + + const actions = el("div", "emdb-field-actions"); + const save = el("button", "emdb-dl", "Save"); + save.title = "Set and remember across sessions"; + save.addEventListener("click", function () { cmd("save", { data_dir: input.value.trim() }); }); + const session = el("button", "emdb-copy-btn", "Use for session"); + session.title = "Set for this session only (not written to disk)"; + session.addEventListener("click", function () { cmd("session", { data_dir: input.value.trim() }); }); + const reset = el("button", "emdb-delete", "Reset"); + reset.title = "Back to the default location"; + reset.addEventListener("click", function () { cmd("reset"); }); + actions.appendChild(save); + actions.appendChild(session); + actions.appendChild(reset); + field.appendChild(actions); + wrap.appendChild(field); + + const status = model.get("status"); + if (status) wrap.appendChild(el("div", "emdb-status", esc(status))); + + // Info: settings file + search order + const info = el("div", "emdb-settings-info"); + info.appendChild(el("div", "emdb-kv", + 'Settings file' + + '' + esc(model.get("config_path") || "") + "")); + + const dirs = model.get("search_dirs") || []; + if (dirs.length) { + info.appendChild(el("div", "emdb-field-label", "Search order")); + const list = el("div", "emdb-searchlist"); + dirs.forEach(function (d, i) { + const last = i === dirs.length - 1; + const tag = last + ? 'downloads here' + : 'shared'; + list.appendChild(el("div", "emdb-searchdir", esc(d) + " " + tag)); + }); + info.appendChild(list); + } + wrap.appendChild(info); + root.appendChild(wrap); + } + + const redraw = function () { draw(); }; + model.on("change:data_dir", redraw); + model.on("change:status", redraw); + model.on("change:search_dirs", redraw); + model.on("change:config_path", redraw); + draw(); + + return function () { + model.off("change:data_dir", redraw); + model.off("change:status", redraw); + model.off("change:search_dirs", redraw); + model.off("change:config_path", redraw); + }; +} + +export default { render }; diff --git a/em_database/tests/conftest.py b/em_database/tests/conftest.py new file mode 100644 index 0000000..68e55c5 --- /dev/null +++ b/em_database/tests/conftest.py @@ -0,0 +1,21 @@ +"""Shared test fixtures. + +Every test runs against an isolated, empty settings file (in a tmp dir) and with +the legacy ``EM_DATABASE_DATA_DIR`` env var cleared, so the developer's real +settings never affect a test and a test never writes to the real config. +""" +import pytest + + +@pytest.fixture(autouse=True) +def _isolate_settings(tmp_path, monkeypatch): + from em_database import config + + monkeypatch.setenv("EM_DATABASE_CONFIG", str(tmp_path / "em_database_settings.yaml")) + # Point the system config at a nonexistent file and clear the shared/data env + # vars, so the developer's real machine config never leaks into a test. + monkeypatch.setenv("EM_DATABASE_SYSTEM_CONFIG", str(tmp_path / "system.yaml")) + monkeypatch.delenv("EM_DATABASE_DATA_DIR", raising=False) + monkeypatch.delenv("EM_DATABASE_SHARED_DIR", raising=False) + config.settings.reload() # re-seed from the isolated (empty) config + yield diff --git a/em_database/tests/test_settings.py b/em_database/tests/test_settings.py new file mode 100644 index 0000000..36b1806 --- /dev/null +++ b/em_database/tests/test_settings.py @@ -0,0 +1,66 @@ +"""Tests for the matplotlib-style settings object and the YAML config file. + +The autouse fixture in conftest.py isolates each test to an empty settings file +and clears the legacy env var, so these exercise the mechanism in isolation. +""" +import em_database +from em_database import config + + +def test_defaults_when_nothing_configured(): + assert em_database.get_data_dir() == config._default_data_dir() + assert em_database.settings["data_dir"] == config._default_data_dir() + + +def test_live_object_is_immediate_like_rcparams(tmp_path): + em_database.settings["data_dir"] = str(tmp_path / "live") + assert em_database.get_data_dir() == str(tmp_path / "live") # no save needed + assert not config.config_path().exists() # not persisted + + +def test_set_data_dir_persists_across_sessions(tmp_path): + target = str(tmp_path / "data") + em_database.set_data_dir(target) # persist=True by default + assert em_database.get_data_dir() == target + assert config._read_file()["data_dir"] == target + config.settings.reload() # a fresh "session" + assert em_database.get_data_dir() == target + + +def test_set_data_dir_session_only_is_not_persisted(tmp_path): + target = str(tmp_path / "x") + em_database.set_data_dir(target, persist=False) + assert em_database.get_data_dir() == target + assert not config.config_path().exists() + config.settings.reload() + assert em_database.get_data_dir() == config._default_data_dir() # forgotten + + +def test_reset_clears_the_saved_choice(tmp_path): + em_database.set_data_dir(str(tmp_path)) + em_database.reset_data_dir() + assert em_database.get_data_dir() == config._default_data_dir() + config.settings.reload() + assert em_database.get_data_dir() == config._default_data_dir() # stays reset + + +def test_generic_setting_roundtrips(): + em_database.set_setting("quality", "high") + assert em_database.get_setting("quality") == "high" + config.settings.reload() + assert em_database.get_setting("quality") == "high" # persisted + + +def test_saving_other_settings_keeps_the_default_dynamic(): + """Persisting an unrelated setting must NOT freeze the default data_dir into + the file - otherwise the default would stop being obeyed.""" + em_database.set_setting("quality", "high") # triggers a save() + stored = config._read_file() + assert stored == {"quality": "high"} # data_dir default not written + assert em_database.get_data_dir() == config._default_data_dir() + + +def test_legacy_env_var_still_seeds(tmp_path, monkeypatch): + monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path / "fromenv")) + config.settings.reload() # a fresh "import" with the env var set + assert em_database.get_data_dir() == str(tmp_path / "fromenv") diff --git a/em_database/tests/test_shared_data.py b/em_database/tests/test_shared_data.py new file mode 100644 index 0000000..7beaf71 --- /dev/null +++ b/em_database/tests/test_shared_data.py @@ -0,0 +1,65 @@ +"""Tests for the ~/.em_database settings folder and system-wide data resolution. + +The conftest fixture isolates the user + system config and clears the shared/data +env vars, so these exercise the resolution logic in isolation (no network - the +"downloads" here find a pre-placed file). +""" +import os + +import em_database +from em_database import catalogue, config +from em_database.tests.test_load_data import TINY_DATASET + + +def _dataset(): + return catalogue.resolve(TINY_DATASET) + + +def test_settings_live_in_dot_em_database(monkeypatch): + monkeypatch.delenv("EM_DATABASE_CONFIG", raising=False) # use the real default + path = config.config_path() + assert path.parent.name == ".em_database" + assert path.name == "settings.yaml" + + +def test_shared_dir_is_searched_before_the_user_dir(tmp_path, monkeypatch): + shared, user = tmp_path / "shared", tmp_path / "user" + shared.mkdir() + user.mkdir() + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared)) + em_database.set_data_dir(str(user), persist=False) + + ds = _dataset() + assert ds.filepath() is None # nowhere yet + (user / ds.file).write_bytes(b"user") + assert ds.filepath() == str(user / ds.file) # found in the user dir + (shared / ds.file).write_bytes(b"shared") + assert ds.filepath() == str(shared / ds.file) # shared/system wins + + +def test_download_uses_a_shared_copy_without_refetching(tmp_path, monkeypatch): + shared, user = tmp_path / "shared", tmp_path / "user" + shared.mkdir() + user.mkdir() + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared)) + em_database.set_data_dir(str(user), persist=False) + + ds = _dataset() + (shared / ds.file).write_bytes(b"payload") # pre-installed system-wide + path = ds.download(background=False) + assert path == str(shared / ds.file) # used the shared copy + assert not (user / ds.file).exists() # nothing downloaded to the user dir + + +def test_search_order_is_shared_then_user(tmp_path, monkeypatch): + a, b = tmp_path / "a", tmp_path / "b" + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", os.pathsep.join([str(a), str(b)])) + em_database.set_data_dir(str(tmp_path / "user"), persist=False) + assert config.data_search_dirs() == [str(a), str(b), str(tmp_path / "user")] + + +def test_system_config_file_contributes_shared_dirs(tmp_path, monkeypatch): + system_file = tmp_path / "system.yaml" + system_file.write_text(f"data_dir: {tmp_path / 'sitewide'}\n", encoding="utf-8") + monkeypatch.setenv("EM_DATABASE_SYSTEM_CONFIG", str(system_file)) + assert str(tmp_path / "sitewide") in config.shared_data_dirs() diff --git a/em_database/tests/test_widget.py b/em_database/tests/test_widget.py index 3c8ad73..489d2c6 100644 --- a/em_database/tests/test_widget.py +++ b/em_database/tests/test_widget.py @@ -42,8 +42,8 @@ def test_catalogue_entry_has_expected_fields(): assert isinstance(row["downloaded"], bool) -def test_catalogue_downloaded_flag_tracks_the_file(tmp_path, monkeypatch): - monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path)) +def test_catalogue_downloaded_flag_tracks_the_file(tmp_path): + em_database.set_data_dir(str(tmp_path), persist=False) ds = catalogue.resolve(TINY_DATASET) assert catalogue.entry(TINY_DATASET, ds)["downloaded"] is False (tmp_path / ds.file).write_bytes(b"x") # pretend it is downloaded @@ -121,8 +121,8 @@ def test_search_blob_includes_authors_and_affiliation(): assert "4d-stem" in row["search"] # technique -def test_delete_removes_downloaded_file(tmp_path, monkeypatch): - monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path)) +def test_delete_removes_downloaded_file(tmp_path): + em_database.set_data_dir(str(tmp_path), persist=False) ds = catalogue.resolve(TINY_DATASET) (tmp_path / ds.file).write_bytes(b"x") assert ds.filepath() is not None @@ -189,6 +189,43 @@ def _boom(_dataset): assert "text/plain" in bundle +def test_settings_widget_edits_and_persists(tmp_path): + pytest.importorskip("anywidget") + from em_database import config + from em_database.widget import settings_widget + + widget = settings_widget() + assert widget.data_dir == em_database.get_data_dir() + + widget._command = {"action": "save", "data_dir": str(tmp_path / "d"), "nonce": 1} + assert em_database.get_data_dir() == str(tmp_path / "d") + assert config._read_file()["data_dir"] == str(tmp_path / "d") # persisted + assert widget.data_dir == str(tmp_path / "d") + + widget._command = {"action": "session", "data_dir": str(tmp_path / "s"), "nonce": 2} + assert em_database.get_data_dir() == str(tmp_path / "s") + assert config._read_file()["data_dir"] == str(tmp_path / "d") # NOT persisted + + widget._command = {"action": "reset", "nonce": 3} + assert em_database.get_data_dir() == config._default_data_dir() + + +def test_settings_display_is_a_widget(): + pytest.importorskip("anywidget") + bundle = em_database.settings._repr_mimebundle_() + mimes = bundle[0] if isinstance(bundle, tuple) else bundle + assert "application/vnd.jupyter.widget-view+json" in mimes + + +def test_notebook_detection_and_colab_enable_are_safe(): + """The frontend helpers must be no-ops off a notebook (e.g. under pytest), + so nothing breaks when em_database is imported in plain Python.""" + import em_database.widget as widget_mod + assert widget_mod._in_notebook() is False + widget_mod._enable_colab_widgets() # must not raise when not on Colab + widget_mod._prepare_frontend() # idempotent, safe + + def test_attach_toast_is_noop_outside_jupyter(): """Outside a Jupyter kernel there is no toast, so a bare download is unaffected.""" @@ -224,8 +261,8 @@ def test_download_toasts_plumbing(): @pytest.mark.slow -def test_widget_download_end_to_end(tmp_path, monkeypatch): - monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path)) +def test_widget_download_end_to_end(tmp_path): + em_database.set_data_dir(str(tmp_path), persist=False) widget = _browser() future = widget._start_download(TINY_DATASET) assert future is not None diff --git a/em_database/widget.py b/em_database/widget.py index 3c4eec6..6672cb3 100644 --- a/em_database/widget.py +++ b/em_database/widget.py @@ -48,6 +48,32 @@ def _quiet_pooch(): pass +_colab_enabled = False + + +def _enable_colab_widgets(): + """On Google Colab, third-party (anywidget) widgets render only once the + custom widget manager is enabled - do it once, automatically. No-op anywhere + else, so widgets work out of the box on Colab and Jupyter alike. + """ + global _colab_enabled + if _colab_enabled: + return + try: + from google.colab import output # importable only on Colab + output.enable_custom_widget_manager() + except Exception: + pass + finally: + _colab_enabled = True + + +def _prepare_frontend(): + """Everything that should happen before a widget renders.""" + _quiet_pooch() + _enable_colab_widgets() + + class DownloadCancelled(Exception): """Raised inside pooch's stream when the user cancels a download. @@ -334,7 +360,7 @@ def card(dataset): This backs ``display(dataset)`` / a dataset being the last line in a cell. Requires anywidget (``pip install em-database[widget]``). """ - _quiet_pooch() + _prepare_frontend() global _card_class if _card_class is None: try: @@ -354,7 +380,7 @@ def browse(**kwargs): every dataset grouped by technique, shows which are downloaded, reveals full metadata on hover, and downloads on click with a live progress toast. """ - _quiet_pooch() + _prepare_frontend() global _browser_class if _browser_class is None: try: @@ -371,6 +397,78 @@ def browse(**kwargs): # Global toasts: a bare ``ds.download()`` in Jupyter pops a cancelable toast # --------------------------------------------------------------------------- +def _make_settings_class(): + """Build the ``SettingsWidget`` class, importing anywidget lazily.""" + import anywidget + import traitlets + + from em_database import config + + class SettingsWidget(anywidget.AnyWidget): + """An editable panel for em_database's settings (the data directory).""" + + _esm = _STATIC / "settings.js" + _css = _STATIC / "browser.css" + + data_dir = traitlets.Unicode().tag(sync=True) + default_dir = traitlets.Unicode().tag(sync=True) + config_path = traitlets.Unicode().tag(sync=True) + search_dirs = traitlets.List().tag(sync=True) + status = traitlets.Unicode().tag(sync=True) + _command = traitlets.Dict().tag(sync=True) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._refresh() + self.observe(self._on_command, names="_command") + + def _refresh(self, status=""): + self.data_dir = config.data_dir() + self.default_dir = config._default_data_dir() + self.config_path = str(config.config_path()) + self.search_dirs = list(config.data_search_dirs()) + self.status = status + + def _on_command(self, change): + command = change.get("new") or {} + action = command.get("action") + value = str(command.get("data_dir", "")).strip() + if action == "save" and value: + config.settings["data_dir"] = value + config.settings.save() + self._refresh(status=f"Saved to {config.config_path()}") + elif action == "session" and value: + config.settings["data_dir"] = value + self._refresh(status="Set for this session (not saved)") + elif action == "reset": + config.settings.reset("data_dir") + self._refresh(status="Reset to the default location") + + return SettingsWidget + + +_settings_class = None + + +def settings_widget(): + """Return an interactive widget for editing em_database's settings (Jupyter). + + Backs ``display(em_database.settings)`` and ``em_database.settings.widget()``. + Requires anywidget (``pip install em-database[widget]``). + """ + _prepare_frontend() + global _settings_class + if _settings_class is None: + try: + _settings_class = _make_settings_class() + except ImportError as error: + raise ImportError( + "The settings widget needs anywidget. Install it with " + "`pip install em-database[widget]` (or `pip install anywidget`)." + ) from error + return _settings_class() + + def _make_toasts_class(): """Build the singleton ``DownloadToasts`` widget, importing anywidget lazily.""" import anywidget @@ -450,19 +548,24 @@ def _set_error(self, token, label, message): _toasts_class = None -def _in_jupyter(): - """True only in a Jupyter kernel (where widgets render), not plain Python.""" +def _in_notebook(): + """True in a notebook frontend that can render widgets (Jupyter, Colab, + VS Code, ...), False in plain Python or a terminal IPython.""" try: from IPython import get_ipython ip = get_ipython() - return ip is not None and ip.__class__.__name__ == "ZMQInteractiveShell" + if ip is None: + return False + # ZMQInteractiveShell = Jupyter; "Shell" = Colab; exclude only the + # terminal shell, which cannot render widgets. + return ip.__class__.__name__ != "TerminalInteractiveShell" except Exception: return False def _get_toasts(): """Return the singleton toasts widget, or None if a toast can't be shown - (not in Jupyter, or anywidget missing). + (not in a notebook, or anywidget missing). The widget is re-displayed on every call so it re-anchors in the current cell: a widget view is tied to a cell's output, so clearing or re-running @@ -471,9 +574,9 @@ def _get_toasts(): never duplicates the toasts. """ global _toasts, _toasts_class - if not _in_jupyter(): + if not _in_notebook(): return None - _quiet_pooch() + _prepare_frontend() try: if _toasts_class is None: _toasts_class = _make_toasts_class() diff --git a/examples/changing_download_location.py b/examples/changing_download_location.py index c847ba7..17b539f 100644 --- a/examples/changing_download_location.py +++ b/examples/changing_download_location.py @@ -1,26 +1,50 @@ """ -Changing the Download Location -============================== +Settings and the Download Location +================================== -This example demonstrates how to change the default download location for datasets -in the `em_database` package. By default, datasets are downloaded to a predefined -directory, but users can customize this location as needed. +Configuration for ``em_database`` lives in a live, matplotlib-``rcParams``-style +object, ``em_database.settings``, seeded at import from a YAML file in +``~/.em_database/settings.yaml``. This example shows how to read it, change the +data directory for the session or for good, and reset it. """ -from em_database import get_data_dir, set_data_dir, reset_data_dir - - - -# Display the current download directory -print("Current download directory:", get_data_dir()) -# Set a new download directory -new_directory = "/path/to/your/custom/directory" # Replace with your desired path -set_data_dir(new_directory) -print("Updated download directory:", get_data_dir()) -# Reset to the default download directory -reset_data_dir() -print("Reset download directory:", get_data_dir()) - - - - +import em_database + +# %% +# The data directory is where datasets download to. It defaults to +# ``~/em_database``. +print("Current download directory:", em_database.get_data_dir()) + +# %% +# Change it for this session only (in memory, not written to disk). +em_database.set_data_dir("/path/to/scratch", persist=False) +print("Session directory:", em_database.get_data_dir()) + +# %% +# Change it and remember the choice across sessions (writes +# ``~/.em_database/settings.yaml``). ``set_data_dir`` persists by default; the +# equivalent low-level form is ``em_database.settings[...] = ...; save()``. +em_database.set_data_dir("/big/disk/em_data") # set + persist +# em_database.settings["data_dir"] = "/big/disk/em_data" # the same thing +# em_database.settings.save() +print("Persisted directory:", em_database.get_data_dir()) + +# %% +# Reset to the default location, forgetting the saved choice. +em_database.reset_data_dir() +print("Reset directory:", em_database.get_data_dir()) + +# %% +# Any setting can be stored, not just the data directory. +em_database.set_setting("quality", "high") +print("quality:", em_database.get_setting("quality")) + +# %% +# Datasets can also be installed system-wide and shared by every user. Point +# ``EM_DATABASE_SHARED_DIR`` (or a ``shared_data_dirs`` setting, or a system +# config file) at the shared location, and ``download()`` / ``filepath()`` will +# find a pre-installed copy there before downloading into your own directory. + +# %% +# Clean up so running this example leaves no settings behind. +em_database.settings.reset()