From 3ad743ff6ac94c4e87ba47bb7ad7a971647b579d Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 20 Aug 2026 11:27:44 -0500 Subject: [PATCH 1/7] Feature: matplotlib-style settings + system-wide data resolution - em_database.settings is a live, rcParams-style dict seeded at import from ~/.em_database/settings.yaml (override with EM_DATABASE_CONFIG). Mutate in memory for an immediate effect; settings.save() / set_data_dir(persist=True) persist across sessions. No env var is needed; EM_DATABASE_DATA_DIR is still honored as a backward-compat seed. - save() writes only values that differ from the defaults, so a default (like the data dir) is never frozen into the file - it stays dynamic and obeyed. - Data can be installed system-wide: filepath()/download() search shared locations first (EM_DATABASE_SHARED_DIR, a shared_data_dirs setting, or a system config at /etc/em_database or %PROGRAMDATA%), then the user's data dir, and only download (into the user dir) if it is nowhere to be found. - Colab: auto-enable the custom widget manager and broaden notebook detection so browse()/display()/toasts work out of the box. - Tests isolate the user + system config; add test_settings and test_shared_data. --- em_database/__init__.py | 42 +++--- em_database/config.py | 198 ++++++++++++++++++++++++++ em_database/downloadable_dataset.py | 55 +++++-- em_database/tests/conftest.py | 21 +++ em_database/tests/test_settings.py | 66 +++++++++ em_database/tests/test_shared_data.py | 65 +++++++++ em_database/tests/test_widget.py | 21 ++- em_database/widget.py | 47 ++++-- 8 files changed, 470 insertions(+), 45 deletions(-) create mode 100644 em_database/config.py create mode 100644 em_database/tests/conftest.py create mode 100644 em_database/tests/test_settings.py create mode 100644 em_database/tests/test_shared_data.py diff --git a/em_database/__init__.py b/em_database/__init__.py index c9f9cc2..adacee5 100644 --- a/em_database/__init__.py +++ b/em_database/__init__.py @@ -1,19 +1,12 @@ ### Example datasets ### import os +from em_database.config import settings from em_database.downloadable_dataset import DownloadableDataset from em_database._create_stubs import build_docstring from em_database import data __all__ = [] -if "EM_DATABASE_DATA_DIR" not in os.environ: - # set the default dir to User's home directory + "/emdata" - os.environ["EM_DATABASE_DATA_DIR"] = os.path.join( - os.path.expanduser("~"),"em_database" - ) - - - def get_data_dir(): """ Get the directory where example datasets are stored. @@ -23,9 +16,10 @@ def get_data_dir(): str Path to the example datasets directory. """ - return os.environ["EM_DATABASE_DATA_DIR"] + from em_database import config + return config.data_dir() -def set_data_dir(path: str): +def set_data_dir(path: str, persist: bool = True): """ Set the directory where example datasets are stored. @@ -33,16 +27,31 @@ def set_data_dir(path: str): ---------- path : str Path to the desired example datasets directory. + persist : bool, optional + If True (the default), remember the choice across sessions by writing it + to the settings file. Pass False for a one-off, in-memory change. """ - os.environ["EM_DATABASE_DATA_DIR"] = path + settings["data_dir"] = str(path) + if persist: + settings.save() def reset_data_dir(): """ - Reset the example datasets directory to the default location. + Reset the example datasets directory to the default location, clearing any + saved choice. """ - os.environ["EM_DATABASE_DATA_DIR"] = os.path.join( - os.path.expanduser("~"),"em_database" - ) + settings.reset("data_dir") + + +def get_setting(key: str, default=None): + """Read a value from :data:`em_database.settings`.""" + return settings.get(key, default) + +def set_setting(key: str, value, persist: bool = True): + """Set a value in :data:`em_database.settings`, persisting it by default.""" + settings[key] = value + if persist: + settings.save() def browse(**kwargs): @@ -60,7 +69,8 @@ def browse(**kwargs): return _browse(**kwargs) -__all__ = ['get_data_dir', 'set_data_dir', 'reset_data_dir', 'browse', "data"] +__all__ = ['get_data_dir', 'set_data_dir', 'reset_data_dir', + 'get_setting', 'set_setting', 'settings', 'browse', "data"] # Let ``display(em_database)`` render the browser. Reassigning the module's diff --git a/em_database/config.py b/em_database/config.py new file mode 100644 index 0000000..3d5a08a --- /dev/null +++ b/em_database/config.py @@ -0,0 +1,198 @@ +"""Runtime settings for em_database, in the style of matplotlib's ``rcParams``. + +``em_database.settings`` is a live dict, seeded at import from a YAML file (or +the built-in defaults). Change it in memory:: + + import em_database + em_database.settings["data_dir"] = "/big/disk/em_data" + +and it takes effect immediately. To remember the choice across sessions, persist +it:: + + em_database.settings.save() + +Settings live in a ``~/.em_database`` folder (``~/.em_database/settings.yaml``), +or wherever ``EM_DATABASE_CONFIG`` points. No environment variable is *needed* to +configure em_database - the settings object is the source of truth; the old +``EM_DATABASE_DATA_DIR`` is still honored as a seed for backward compatibility. + +Data can also be installed **system-wide** and shared by every user. When +resolving where a dataset lives, em_database looks in the shared/system +locations first, then the user's own data directory, and only downloads (into +the user directory) if it is nowhere to be found. Shared locations come from a +system config file (see :func:`system_config_path`), the ``EM_DATABASE_SHARED_DIR`` +environment variable (an ``os.pathsep``-separated list), or a ``shared_data_dirs`` +list in the user's settings. +""" +from __future__ import annotations + +import os +from pathlib import Path + +import yaml + + +def _default_data_dir() -> str: + return os.path.join(os.path.expanduser("~"), "em_database") + + +#: Built-in defaults - the fallback when nothing is configured. +DEFAULTS: dict = { + "data_dir": _default_data_dir(), +} + + +def config_dir() -> Path: + """The ``~/.em_database`` folder that holds the settings file.""" + return Path(os.path.expanduser("~")) / ".em_database" + + +def config_path() -> Path: + """Path to the YAML settings file. + + ``EM_DATABASE_CONFIG`` overrides the location; otherwise it is + ``~/.em_database/settings.yaml``. + """ + override = os.environ.get("EM_DATABASE_CONFIG") + if override: + return Path(override) + return config_dir() / "settings.yaml" + + +def system_config_path() -> Path | None: + """Path to the machine-wide settings file (for system-wide installs). + + ``EM_DATABASE_SYSTEM_CONFIG`` overrides it; otherwise it is + ``/etc/em_database/settings.yaml`` (POSIX) or + ``%PROGRAMDATA%\\em_database\\settings.yaml`` (Windows). + """ + override = os.environ.get("EM_DATABASE_SYSTEM_CONFIG") + if override: + return Path(override) + if os.name == "nt": + base = os.environ.get("PROGRAMDATA") + return Path(base) / "em_database" / "settings.yaml" if base else None + return Path("/etc/em_database/settings.yaml") + + +def _read_yaml(path: Path | None) -> dict: + """Parse a YAML file into a dict, or ``{}`` if missing/unreadable/malformed.""" + if path is None: + return {} + try: + with open(path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) + return data if isinstance(data, dict) else {} + except FileNotFoundError: + return {} + except Exception: + # A malformed settings file must not break importing the package. + return {} + + +def _read_file() -> dict: + """The user settings stored on disk, or ``{}``.""" + return _read_yaml(config_path()) + + +def _read_system() -> dict: + """The machine-wide settings, or ``{}``.""" + return _read_yaml(system_config_path()) + + +def _write_file(data: dict) -> None: + path = config_path() + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + yaml.safe_dump(dict(data), handle, default_flow_style=False, sort_keys=True) + + +def _seed(settings: "Settings") -> None: + """(Re)seed a settings object: defaults < file < legacy env var.""" + settings.clear() + settings.update(DEFAULTS) + settings.update(_read_file()) # the file overrides the defaults + # Backward compatibility: seed from the old env var if it is set. + env_dir = os.environ.get("EM_DATABASE_DATA_DIR") + if env_dir: + settings["data_dir"] = env_dir + + +class Settings(dict): + """A live settings mapping that can persist itself to the YAML file.""" + + def save(self) -> None: + """Persist the settings to the YAML file. + + Only values that DIFFER from the built-in defaults are written, so a + default (like the data directory) is never frozen into the file: it stays + dynamic and keeps being obeyed across sessions, machines, or a change to + the default itself. When nothing differs, any stale file is removed. + """ + delta = {key: value for key, value in self.items() if DEFAULTS.get(key) != value} + path = config_path() + if delta: + _write_file(delta) + elif path.exists(): + path.unlink() + + def reload(self) -> None: + """Re-seed from the built-in defaults and the on-disk file, discarding + any in-memory changes.""" + _seed(self) + + def reset(self, key: str | None = None) -> None: + """Reset a key (or everything) to its default, in memory and on disk.""" + if key is None: + self.clear() + self.update(DEFAULTS) + elif key in DEFAULTS: + self[key] = DEFAULTS[key] + else: + self.pop(key, None) + self.save() # delta-save drops the now-default key from the file + + +#: The live settings object (seeded once at import). +settings = Settings() +_seed(settings) + + +def data_dir() -> str: + """The user's data directory - where downloads are written.""" + return str(settings.get("data_dir") or _default_data_dir()) + + +def shared_data_dirs() -> list[str]: + """System-wide / shared data locations, checked before the user's dir. + + Order: ``EM_DATABASE_SHARED_DIR`` (an ``os.pathsep`` list), then the user's + ``shared_data_dirs`` setting, then the system config file's ``data_dir`` and + ``shared_data_dirs``. Duplicates are removed, order preserved. + """ + dirs: list[str] = [] + env = os.environ.get("EM_DATABASE_SHARED_DIR") + if env: + dirs += [d for d in env.split(os.pathsep) if d] + dirs += [str(d) for d in (settings.get("shared_data_dirs") or [])] + system = _read_system() + if system.get("data_dir"): + dirs.append(str(system["data_dir"])) + dirs += [str(d) for d in (system.get("shared_data_dirs") or [])] + seen: set[str] = set() + unique: list[str] = [] + for d in dirs: + if d not in seen: + seen.add(d) + unique.append(d) + return unique + + +def data_search_dirs() -> list[str]: + """Everywhere to look for an existing dataset: shared/system dirs first, + then the user's data directory.""" + dirs = shared_data_dirs() + user = data_dir() + if user not in dirs: + dirs.append(user) + return dirs diff --git a/em_database/downloadable_dataset.py b/em_database/downloadable_dataset.py index e07c9f0..ad7cfbe 100644 --- a/em_database/downloadable_dataset.py +++ b/em_database/downloadable_dataset.py @@ -129,8 +129,8 @@ def _repr_mimebundle_(self, **kwargs): def _resolve_destination(destination: str | None) -> str: """Return the directory the dataset should live in.""" if destination is None: - return os.environ.get("EM_DATABASE_DATA_DIR", - os.path.join(os.path.expanduser("~"), "em_database")) + from em_database import config + return config.data_dir() return destination def download(self, @@ -176,7 +176,12 @@ def download(self, """ if not background: return self._retrieve(destination, progressbar, chunk_size) - target = os.path.join(self._resolve_destination(destination), self.file) + # Resolve where the file will end up: an existing shared/user copy if it + # is already present, otherwise the user's download location. + if destination is not None: + target = os.path.join(destination, self.file) + else: + target = self.filepath() or os.path.join(self._resolve_destination(None), self.file) # In Jupyter (with the widget installed) a background download pops a # cancelable toast; the toast's monitor replaces the plain progress bar. monitor = finish = None @@ -198,14 +203,25 @@ def _retrieve(self, destination: str | None = None, progressbar: bool = True, chunk_size: int = 4096) -> str: - """Fetch the file with pooch and return its local path (blocking).""" + """Fetch the file and return its local path (blocking). + + With no explicit destination, an existing system-wide/shared copy is used + as-is (never re-downloaded); otherwise pooch downloads into the user's + data directory. + """ if progressbar: try: import tqdm # noqa: F401 except ImportError: print("`tqdm` is not installed, progress bar will be disabled.") progressbar = False - destination = self._resolve_destination(destination) + if destination is None: + shared = self._find_shared() + if shared is not None: + return shared + destination = self._resolve_destination(None) + else: + destination = self._resolve_destination(destination) # Instantiate an Http downloader with a custom user agent headers = {"User-Agent": "em_database (https://github.com/CSSFrancis/em_data)"} downloader = pooch.HTTPDownloader(progressbar=progressbar, @@ -220,17 +236,26 @@ def _retrieve(self, ) return filepath + def _find_shared(self) -> str | None: + """Path to an existing copy in a shared/system data dir, or None.""" + from em_database import config + for directory in config.shared_data_dirs(): + candidate = os.path.join(directory, self.file) + if os.path.exists(candidate): + return candidate + return None + def filepath(self) -> str: - """ Return the local file path of the dataset if downloaded. - - If not downloaded return None. """ - destination = os.environ.get("EM_DATABASE_DATA_DIR", - os.path.join(os.path.expanduser("~"), "em_database")) - filepath = os.path.join(destination, self.file) - if os.path.exists(filepath): - return filepath - else: - return None + """ Return the local file path of the dataset if present. + + Looks in the shared/system data locations first, then the user's data + directory. Returns None if the dataset is not downloaded anywhere. """ + from em_database import config + for directory in config.data_search_dirs(): + candidate = os.path.join(directory, self.file) + if os.path.exists(candidate): + return candidate + return None def delete(self, destination: str | None = None) -> bool: """ Delete the downloaded file if it is present. diff --git a/em_database/tests/conftest.py b/em_database/tests/conftest.py new file mode 100644 index 0000000..68e55c5 --- /dev/null +++ b/em_database/tests/conftest.py @@ -0,0 +1,21 @@ +"""Shared test fixtures. + +Every test runs against an isolated, empty settings file (in a tmp dir) and with +the legacy ``EM_DATABASE_DATA_DIR`` env var cleared, so the developer's real +settings never affect a test and a test never writes to the real config. +""" +import pytest + + +@pytest.fixture(autouse=True) +def _isolate_settings(tmp_path, monkeypatch): + from em_database import config + + monkeypatch.setenv("EM_DATABASE_CONFIG", str(tmp_path / "em_database_settings.yaml")) + # Point the system config at a nonexistent file and clear the shared/data env + # vars, so the developer's real machine config never leaks into a test. + monkeypatch.setenv("EM_DATABASE_SYSTEM_CONFIG", str(tmp_path / "system.yaml")) + monkeypatch.delenv("EM_DATABASE_DATA_DIR", raising=False) + monkeypatch.delenv("EM_DATABASE_SHARED_DIR", raising=False) + config.settings.reload() # re-seed from the isolated (empty) config + yield diff --git a/em_database/tests/test_settings.py b/em_database/tests/test_settings.py new file mode 100644 index 0000000..36b1806 --- /dev/null +++ b/em_database/tests/test_settings.py @@ -0,0 +1,66 @@ +"""Tests for the matplotlib-style settings object and the YAML config file. + +The autouse fixture in conftest.py isolates each test to an empty settings file +and clears the legacy env var, so these exercise the mechanism in isolation. +""" +import em_database +from em_database import config + + +def test_defaults_when_nothing_configured(): + assert em_database.get_data_dir() == config._default_data_dir() + assert em_database.settings["data_dir"] == config._default_data_dir() + + +def test_live_object_is_immediate_like_rcparams(tmp_path): + em_database.settings["data_dir"] = str(tmp_path / "live") + assert em_database.get_data_dir() == str(tmp_path / "live") # no save needed + assert not config.config_path().exists() # not persisted + + +def test_set_data_dir_persists_across_sessions(tmp_path): + target = str(tmp_path / "data") + em_database.set_data_dir(target) # persist=True by default + assert em_database.get_data_dir() == target + assert config._read_file()["data_dir"] == target + config.settings.reload() # a fresh "session" + assert em_database.get_data_dir() == target + + +def test_set_data_dir_session_only_is_not_persisted(tmp_path): + target = str(tmp_path / "x") + em_database.set_data_dir(target, persist=False) + assert em_database.get_data_dir() == target + assert not config.config_path().exists() + config.settings.reload() + assert em_database.get_data_dir() == config._default_data_dir() # forgotten + + +def test_reset_clears_the_saved_choice(tmp_path): + em_database.set_data_dir(str(tmp_path)) + em_database.reset_data_dir() + assert em_database.get_data_dir() == config._default_data_dir() + config.settings.reload() + assert em_database.get_data_dir() == config._default_data_dir() # stays reset + + +def test_generic_setting_roundtrips(): + em_database.set_setting("quality", "high") + assert em_database.get_setting("quality") == "high" + config.settings.reload() + assert em_database.get_setting("quality") == "high" # persisted + + +def test_saving_other_settings_keeps_the_default_dynamic(): + """Persisting an unrelated setting must NOT freeze the default data_dir into + the file - otherwise the default would stop being obeyed.""" + em_database.set_setting("quality", "high") # triggers a save() + stored = config._read_file() + assert stored == {"quality": "high"} # data_dir default not written + assert em_database.get_data_dir() == config._default_data_dir() + + +def test_legacy_env_var_still_seeds(tmp_path, monkeypatch): + monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path / "fromenv")) + config.settings.reload() # a fresh "import" with the env var set + assert em_database.get_data_dir() == str(tmp_path / "fromenv") diff --git a/em_database/tests/test_shared_data.py b/em_database/tests/test_shared_data.py new file mode 100644 index 0000000..7beaf71 --- /dev/null +++ b/em_database/tests/test_shared_data.py @@ -0,0 +1,65 @@ +"""Tests for the ~/.em_database settings folder and system-wide data resolution. + +The conftest fixture isolates the user + system config and clears the shared/data +env vars, so these exercise the resolution logic in isolation (no network - the +"downloads" here find a pre-placed file). +""" +import os + +import em_database +from em_database import catalogue, config +from em_database.tests.test_load_data import TINY_DATASET + + +def _dataset(): + return catalogue.resolve(TINY_DATASET) + + +def test_settings_live_in_dot_em_database(monkeypatch): + monkeypatch.delenv("EM_DATABASE_CONFIG", raising=False) # use the real default + path = config.config_path() + assert path.parent.name == ".em_database" + assert path.name == "settings.yaml" + + +def test_shared_dir_is_searched_before_the_user_dir(tmp_path, monkeypatch): + shared, user = tmp_path / "shared", tmp_path / "user" + shared.mkdir() + user.mkdir() + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared)) + em_database.set_data_dir(str(user), persist=False) + + ds = _dataset() + assert ds.filepath() is None # nowhere yet + (user / ds.file).write_bytes(b"user") + assert ds.filepath() == str(user / ds.file) # found in the user dir + (shared / ds.file).write_bytes(b"shared") + assert ds.filepath() == str(shared / ds.file) # shared/system wins + + +def test_download_uses_a_shared_copy_without_refetching(tmp_path, monkeypatch): + shared, user = tmp_path / "shared", tmp_path / "user" + shared.mkdir() + user.mkdir() + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared)) + em_database.set_data_dir(str(user), persist=False) + + ds = _dataset() + (shared / ds.file).write_bytes(b"payload") # pre-installed system-wide + path = ds.download(background=False) + assert path == str(shared / ds.file) # used the shared copy + assert not (user / ds.file).exists() # nothing downloaded to the user dir + + +def test_search_order_is_shared_then_user(tmp_path, monkeypatch): + a, b = tmp_path / "a", tmp_path / "b" + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", os.pathsep.join([str(a), str(b)])) + em_database.set_data_dir(str(tmp_path / "user"), persist=False) + assert config.data_search_dirs() == [str(a), str(b), str(tmp_path / "user")] + + +def test_system_config_file_contributes_shared_dirs(tmp_path, monkeypatch): + system_file = tmp_path / "system.yaml" + system_file.write_text(f"data_dir: {tmp_path / 'sitewide'}\n", encoding="utf-8") + monkeypatch.setenv("EM_DATABASE_SYSTEM_CONFIG", str(system_file)) + assert str(tmp_path / "sitewide") in config.shared_data_dirs() diff --git a/em_database/tests/test_widget.py b/em_database/tests/test_widget.py index 3c8ad73..6595188 100644 --- a/em_database/tests/test_widget.py +++ b/em_database/tests/test_widget.py @@ -42,8 +42,8 @@ def test_catalogue_entry_has_expected_fields(): assert isinstance(row["downloaded"], bool) -def test_catalogue_downloaded_flag_tracks_the_file(tmp_path, monkeypatch): - monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path)) +def test_catalogue_downloaded_flag_tracks_the_file(tmp_path): + em_database.set_data_dir(str(tmp_path), persist=False) ds = catalogue.resolve(TINY_DATASET) assert catalogue.entry(TINY_DATASET, ds)["downloaded"] is False (tmp_path / ds.file).write_bytes(b"x") # pretend it is downloaded @@ -121,8 +121,8 @@ def test_search_blob_includes_authors_and_affiliation(): assert "4d-stem" in row["search"] # technique -def test_delete_removes_downloaded_file(tmp_path, monkeypatch): - monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path)) +def test_delete_removes_downloaded_file(tmp_path): + em_database.set_data_dir(str(tmp_path), persist=False) ds = catalogue.resolve(TINY_DATASET) (tmp_path / ds.file).write_bytes(b"x") assert ds.filepath() is not None @@ -189,6 +189,15 @@ def _boom(_dataset): assert "text/plain" in bundle +def test_notebook_detection_and_colab_enable_are_safe(): + """The frontend helpers must be no-ops off a notebook (e.g. under pytest), + so nothing breaks when em_database is imported in plain Python.""" + import em_database.widget as widget_mod + assert widget_mod._in_notebook() is False + widget_mod._enable_colab_widgets() # must not raise when not on Colab + widget_mod._prepare_frontend() # idempotent, safe + + def test_attach_toast_is_noop_outside_jupyter(): """Outside a Jupyter kernel there is no toast, so a bare download is unaffected.""" @@ -224,8 +233,8 @@ def test_download_toasts_plumbing(): @pytest.mark.slow -def test_widget_download_end_to_end(tmp_path, monkeypatch): - monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path)) +def test_widget_download_end_to_end(tmp_path): + em_database.set_data_dir(str(tmp_path), persist=False) widget = _browser() future = widget._start_download(TINY_DATASET) assert future is not None diff --git a/em_database/widget.py b/em_database/widget.py index 3c4eec6..ecee874 100644 --- a/em_database/widget.py +++ b/em_database/widget.py @@ -48,6 +48,32 @@ def _quiet_pooch(): pass +_colab_enabled = False + + +def _enable_colab_widgets(): + """On Google Colab, third-party (anywidget) widgets render only once the + custom widget manager is enabled - do it once, automatically. No-op anywhere + else, so widgets work out of the box on Colab and Jupyter alike. + """ + global _colab_enabled + if _colab_enabled: + return + try: + from google.colab import output # importable only on Colab + output.enable_custom_widget_manager() + except Exception: + pass + finally: + _colab_enabled = True + + +def _prepare_frontend(): + """Everything that should happen before a widget renders.""" + _quiet_pooch() + _enable_colab_widgets() + + class DownloadCancelled(Exception): """Raised inside pooch's stream when the user cancels a download. @@ -334,7 +360,7 @@ def card(dataset): This backs ``display(dataset)`` / a dataset being the last line in a cell. Requires anywidget (``pip install em-database[widget]``). """ - _quiet_pooch() + _prepare_frontend() global _card_class if _card_class is None: try: @@ -354,7 +380,7 @@ def browse(**kwargs): every dataset grouped by technique, shows which are downloaded, reveals full metadata on hover, and downloads on click with a live progress toast. """ - _quiet_pooch() + _prepare_frontend() global _browser_class if _browser_class is None: try: @@ -450,19 +476,24 @@ def _set_error(self, token, label, message): _toasts_class = None -def _in_jupyter(): - """True only in a Jupyter kernel (where widgets render), not plain Python.""" +def _in_notebook(): + """True in a notebook frontend that can render widgets (Jupyter, Colab, + VS Code, ...), False in plain Python or a terminal IPython.""" try: from IPython import get_ipython ip = get_ipython() - return ip is not None and ip.__class__.__name__ == "ZMQInteractiveShell" + if ip is None: + return False + # ZMQInteractiveShell = Jupyter; "Shell" = Colab; exclude only the + # terminal shell, which cannot render widgets. + return ip.__class__.__name__ != "TerminalInteractiveShell" except Exception: return False def _get_toasts(): """Return the singleton toasts widget, or None if a toast can't be shown - (not in Jupyter, or anywidget missing). + (not in a notebook, or anywidget missing). The widget is re-displayed on every call so it re-anchors in the current cell: a widget view is tied to a cell's output, so clearing or re-running @@ -471,9 +502,9 @@ def _get_toasts(): never duplicates the toasts. """ global _toasts, _toasts_class - if not _in_jupyter(): + if not _in_notebook(): return None - _quiet_pooch() + _prepare_frontend() try: if _toasts_class is None: _toasts_class = _make_toasts_class() From 0cc33001ebbf67428406129cd10f1aeb056f840e Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 20 Aug 2026 12:07:08 -0500 Subject: [PATCH 2/7] Docs: document the settings model and shared data Update the README and the changing_download_location example to use em_database.settings (matplotlib rcParams-style, ~/.em_database/settings.yaml), the persist/session-only set_data_dir, generic get_setting/set_setting, and the system-wide shared-data locations. The example resets settings at the end so it leaves nothing behind when run by the docs gallery. --- README.md | 39 +++++++++++++-- examples/changing_download_location.py | 68 +++++++++++++++++--------- 2 files changed, 80 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 7879fef..78e4088 100644 --- a/README.md +++ b/README.md @@ -35,17 +35,46 @@ Call `path.done()` to check progress without blocking, or `path.result()` to wai explicitly. Pass `download(background=False)` to block and get the path as a plain string instead. -The download location defaults to `~/em_database` and can be changed either for the -session or through the `EM_DATABASE_DATA_DIR` environment variable: +## Settings + +Configuration lives in a live, matplotlib-`rcParams`-style object, +`em_database.settings`, seeded at import from `~/.em_database/settings.yaml`. +Change it in memory for an immediate effect, and persist it to remember the +choice across sessions: ```python import em_database -em_database.set_data_dir("/path/to/somewhere") -em_database.get_data_dir() -em_database.reset_data_dir() +em_database.settings["data_dir"] = "/big/disk/em_data" # takes effect now +em_database.settings.save() # remember it next time +``` + +The data directory defaults to `~/em_database`. Convenience helpers wrap the +common case — `set_data_dir` persists by default: + +```python +em_database.get_data_dir() # current location +em_database.set_data_dir("/big/disk/em_data") # set + persist +em_database.set_data_dir("/scratch", persist=False) # one-off, in-memory only +em_database.reset_data_dir() # back to the default, forget the choice ``` +No environment variable is needed. Only values you explicitly set are written to +the file, so the default is never frozen in — it keeps being obeyed even if it +changes. For a one-off override (e.g. CI) the legacy `EM_DATABASE_DATA_DIR` is +still honored, and `EM_DATABASE_CONFIG` relocates the settings file. + +### Shared, system-wide data + +Datasets can be installed once for every user. `download()` and `filepath()` +look in the shared/system locations **first**, then your own data directory, and +only download (into your directory) if the file is nowhere to be found — so a +shared copy is reused instead of refetched. Shared locations come from the +`EM_DATABASE_SHARED_DIR` environment variable (an `os.pathsep`-separated list), a +`shared_data_dirs` list in your settings, or a system config file +(`/etc/em_database/settings.yaml`, or `%PROGRAMDATA%\em_database\settings.yaml` +on Windows). + ## Adding a dataset Datasets are described by a YAML file in `em_database/datasets/`, one entry per file, diff --git a/examples/changing_download_location.py b/examples/changing_download_location.py index c847ba7..17b539f 100644 --- a/examples/changing_download_location.py +++ b/examples/changing_download_location.py @@ -1,26 +1,50 @@ """ -Changing the Download Location -============================== +Settings and the Download Location +================================== -This example demonstrates how to change the default download location for datasets -in the `em_database` package. By default, datasets are downloaded to a predefined -directory, but users can customize this location as needed. +Configuration for ``em_database`` lives in a live, matplotlib-``rcParams``-style +object, ``em_database.settings``, seeded at import from a YAML file in +``~/.em_database/settings.yaml``. This example shows how to read it, change the +data directory for the session or for good, and reset it. """ -from em_database import get_data_dir, set_data_dir, reset_data_dir - - - -# Display the current download directory -print("Current download directory:", get_data_dir()) -# Set a new download directory -new_directory = "/path/to/your/custom/directory" # Replace with your desired path -set_data_dir(new_directory) -print("Updated download directory:", get_data_dir()) -# Reset to the default download directory -reset_data_dir() -print("Reset download directory:", get_data_dir()) - - - - +import em_database + +# %% +# The data directory is where datasets download to. It defaults to +# ``~/em_database``. +print("Current download directory:", em_database.get_data_dir()) + +# %% +# Change it for this session only (in memory, not written to disk). +em_database.set_data_dir("/path/to/scratch", persist=False) +print("Session directory:", em_database.get_data_dir()) + +# %% +# Change it and remember the choice across sessions (writes +# ``~/.em_database/settings.yaml``). ``set_data_dir`` persists by default; the +# equivalent low-level form is ``em_database.settings[...] = ...; save()``. +em_database.set_data_dir("/big/disk/em_data") # set + persist +# em_database.settings["data_dir"] = "/big/disk/em_data" # the same thing +# em_database.settings.save() +print("Persisted directory:", em_database.get_data_dir()) + +# %% +# Reset to the default location, forgetting the saved choice. +em_database.reset_data_dir() +print("Reset directory:", em_database.get_data_dir()) + +# %% +# Any setting can be stored, not just the data directory. +em_database.set_setting("quality", "high") +print("quality:", em_database.get_setting("quality")) + +# %% +# Datasets can also be installed system-wide and shared by every user. Point +# ``EM_DATABASE_SHARED_DIR`` (or a ``shared_data_dirs`` setting, or a system +# config file) at the shared location, and ``download()`` / ``filepath()`` will +# find a pre-installed copy there before downloading into your own directory. + +# %% +# Clean up so running this example leaves no settings behind. +em_database.settings.reset() From b6c60cf100b6c1b12801fa4545d835f06f3c6c1d Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 20 Aug 2026 12:56:45 -0500 Subject: [PATCH 3/7] Feature: an editable settings widget display(em_database.settings) (or em_database.settings.widget()) now renders an anywidget panel to set the data directory directly - Save (persist), Use for session (in-memory), or Reset - and shows the settings-file path and the shared-then-user search order. Falls back to the dict repr without anywidget. --- README.md | 3 + em_database/config.py | 15 +++++ em_database/static/browser.css | 23 +++++++ em_database/static/settings.js | 101 +++++++++++++++++++++++++++++++ em_database/tests/test_widget.py | 28 +++++++++ em_database/widget.py | 72 ++++++++++++++++++++++ 6 files changed, 242 insertions(+) create mode 100644 em_database/static/settings.js diff --git a/README.md b/README.md index 78e4088..9fe94d1 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ em_database.settings["data_dir"] = "/big/disk/em_data" # takes effect now em_database.settings.save() # remember it next time ``` +In Jupyter you can also edit them interactively — `display(em_database.settings)` +renders a panel to set (and save) the data directory directly. + The data directory defaults to `~/em_database`. Convenience helpers wrap the common case — `set_data_dir` persists by default: diff --git a/em_database/config.py b/em_database/config.py index 3d5a08a..1fa7bb3 100644 --- a/em_database/config.py +++ b/em_database/config.py @@ -152,6 +152,21 @@ def reset(self, key: str | None = None) -> None: self.pop(key, None) self.save() # delta-save drops the now-default key from the file + def widget(self): + """Return an interactive widget for editing the settings (Jupyter).""" + from em_database.widget import settings_widget + return settings_widget() + + def _repr_mimebundle_(self, **kwargs): + """Render as an editable settings widget in Jupyter (falls back to the + plain dict repr if anywidget is not installed).""" + try: + from em_database.widget import settings_widget + widget = settings_widget() + except Exception: + return {"text/plain": repr(dict(self))} + return widget._repr_mimebundle_(**kwargs) + #: The live settings object (seeded once at import). settings = Settings() diff --git a/em_database/static/browser.css b/em_database/static/browser.css index b4eb66d..818df0e 100644 --- a/em_database/static/browser.css +++ b/em_database/static/browser.css @@ -196,6 +196,29 @@ .emdb-inline-toast.cancelling { opacity: 0.65; } .emdb-inline-toast.error { border-color: var(--emdb-red); } +/* Settings panel (display(em_database.settings)) ------------------- */ +.emdb-settings-host { max-width: 620px; } +.emdb-settings { padding: 16px 18px; } +.emdb-field { margin-top: 12px; } +.emdb-field-label { + font-size: 10.5px; font-weight: 700; letter-spacing: 0.06em; + text-transform: uppercase; color: var(--emdb-muted); margin: 12px 0 6px; +} +.emdb-field-actions { display: flex; gap: 8px; margin-top: 8px; } +.emdb-status { margin-top: 12px; font-size: 12px; color: var(--emdb-green); font-weight: 600; } +.emdb-settings-info { margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--emdb-surface0); } +.emdb-searchlist { display: flex; flex-direction: column; gap: 5px; } +.emdb-searchdir { + font-family: var(--emdb-mono); font-size: 11.5px; color: var(--emdb-text); + word-break: break-all; +} +.emdb-tag { + font-family: var(--emdb-font); font-size: 10px; font-weight: 600; + color: var(--emdb-crust); background: var(--emdb-green); + border-radius: 4px; padding: 1px 6px; white-space: nowrap; +} +.emdb-tag.shared { background: var(--emdb-blue); } + /* Toasts: pinned to the bottom-right of the viewport ----------------- */ .emdb-toast-root { position: fixed; right: 16px; bottom: 16px; z-index: 99999; diff --git a/em_database/static/settings.js b/em_database/static/settings.js new file mode 100644 index 0000000..72e9767 --- /dev/null +++ b/em_database/static/settings.js @@ -0,0 +1,101 @@ +// settings.js - an editable settings panel (what display(em_database.settings) shows). +// +// Edit the data directory directly: type a path and Save (persist), Use for +// session (in-memory), or Reset to the default. Also shows the settings-file +// location and the shared-then-user search order. + +function esc(value) { + return String(value).replace(/[&<>"']/g, function (c) { + return { "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]; + }); +} + +function el(tag, cls, html) { + const n = document.createElement(tag); + if (cls) n.className = cls; + if (html != null) n.innerHTML = html; + return n; +} + +function render({ model, el: root }) { + root.classList.add("emdb", "emdb-settings-host"); + + let nonce = 0; + function cmd(action, extra) { + model.set("_command", Object.assign({ action, nonce: nonce++ }, extra || {})); + model.save_changes(); + } + + function draw() { + root.innerHTML = ""; + const wrap = el("div", "emdb-settings"); + wrap.appendChild(el("div", "emdb-brand", '◆ Settings')); + + // Data directory field + const field = el("div", "emdb-field"); + field.appendChild(el("div", "emdb-field-label", "Data directory")); + const input = el("input", "emdb-search"); + input.type = "text"; + input.value = model.get("data_dir") || ""; + input.placeholder = model.get("default_dir") || ""; + input.spellcheck = false; + field.appendChild(input); + + const actions = el("div", "emdb-field-actions"); + const save = el("button", "emdb-dl", "Save"); + save.title = "Set and remember across sessions"; + save.addEventListener("click", function () { cmd("save", { data_dir: input.value.trim() }); }); + const session = el("button", "emdb-copy-btn", "Use for session"); + session.title = "Set for this session only (not written to disk)"; + session.addEventListener("click", function () { cmd("session", { data_dir: input.value.trim() }); }); + const reset = el("button", "emdb-delete", "Reset"); + reset.title = "Back to the default location"; + reset.addEventListener("click", function () { cmd("reset"); }); + actions.appendChild(save); + actions.appendChild(session); + actions.appendChild(reset); + field.appendChild(actions); + wrap.appendChild(field); + + const status = model.get("status"); + if (status) wrap.appendChild(el("div", "emdb-status", esc(status))); + + // Info: settings file + search order + const info = el("div", "emdb-settings-info"); + info.appendChild(el("div", "emdb-kv", + 'Settings file' + + '' + esc(model.get("config_path") || "") + "")); + + const dirs = model.get("search_dirs") || []; + if (dirs.length) { + info.appendChild(el("div", "emdb-field-label", "Search order")); + const list = el("div", "emdb-searchlist"); + dirs.forEach(function (d, i) { + const last = i === dirs.length - 1; + const tag = last + ? 'downloads here' + : 'shared'; + list.appendChild(el("div", "emdb-searchdir", esc(d) + " " + tag)); + }); + info.appendChild(list); + } + wrap.appendChild(info); + root.appendChild(wrap); + } + + const redraw = function () { draw(); }; + model.on("change:data_dir", redraw); + model.on("change:status", redraw); + model.on("change:search_dirs", redraw); + model.on("change:config_path", redraw); + draw(); + + return function () { + model.off("change:data_dir", redraw); + model.off("change:status", redraw); + model.off("change:search_dirs", redraw); + model.off("change:config_path", redraw); + }; +} + +export default { render }; diff --git a/em_database/tests/test_widget.py b/em_database/tests/test_widget.py index 6595188..489d2c6 100644 --- a/em_database/tests/test_widget.py +++ b/em_database/tests/test_widget.py @@ -189,6 +189,34 @@ def _boom(_dataset): assert "text/plain" in bundle +def test_settings_widget_edits_and_persists(tmp_path): + pytest.importorskip("anywidget") + from em_database import config + from em_database.widget import settings_widget + + widget = settings_widget() + assert widget.data_dir == em_database.get_data_dir() + + widget._command = {"action": "save", "data_dir": str(tmp_path / "d"), "nonce": 1} + assert em_database.get_data_dir() == str(tmp_path / "d") + assert config._read_file()["data_dir"] == str(tmp_path / "d") # persisted + assert widget.data_dir == str(tmp_path / "d") + + widget._command = {"action": "session", "data_dir": str(tmp_path / "s"), "nonce": 2} + assert em_database.get_data_dir() == str(tmp_path / "s") + assert config._read_file()["data_dir"] == str(tmp_path / "d") # NOT persisted + + widget._command = {"action": "reset", "nonce": 3} + assert em_database.get_data_dir() == config._default_data_dir() + + +def test_settings_display_is_a_widget(): + pytest.importorskip("anywidget") + bundle = em_database.settings._repr_mimebundle_() + mimes = bundle[0] if isinstance(bundle, tuple) else bundle + assert "application/vnd.jupyter.widget-view+json" in mimes + + def test_notebook_detection_and_colab_enable_are_safe(): """The frontend helpers must be no-ops off a notebook (e.g. under pytest), so nothing breaks when em_database is imported in plain Python.""" diff --git a/em_database/widget.py b/em_database/widget.py index ecee874..6672cb3 100644 --- a/em_database/widget.py +++ b/em_database/widget.py @@ -397,6 +397,78 @@ def browse(**kwargs): # Global toasts: a bare ``ds.download()`` in Jupyter pops a cancelable toast # --------------------------------------------------------------------------- +def _make_settings_class(): + """Build the ``SettingsWidget`` class, importing anywidget lazily.""" + import anywidget + import traitlets + + from em_database import config + + class SettingsWidget(anywidget.AnyWidget): + """An editable panel for em_database's settings (the data directory).""" + + _esm = _STATIC / "settings.js" + _css = _STATIC / "browser.css" + + data_dir = traitlets.Unicode().tag(sync=True) + default_dir = traitlets.Unicode().tag(sync=True) + config_path = traitlets.Unicode().tag(sync=True) + search_dirs = traitlets.List().tag(sync=True) + status = traitlets.Unicode().tag(sync=True) + _command = traitlets.Dict().tag(sync=True) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._refresh() + self.observe(self._on_command, names="_command") + + def _refresh(self, status=""): + self.data_dir = config.data_dir() + self.default_dir = config._default_data_dir() + self.config_path = str(config.config_path()) + self.search_dirs = list(config.data_search_dirs()) + self.status = status + + def _on_command(self, change): + command = change.get("new") or {} + action = command.get("action") + value = str(command.get("data_dir", "")).strip() + if action == "save" and value: + config.settings["data_dir"] = value + config.settings.save() + self._refresh(status=f"Saved to {config.config_path()}") + elif action == "session" and value: + config.settings["data_dir"] = value + self._refresh(status="Set for this session (not saved)") + elif action == "reset": + config.settings.reset("data_dir") + self._refresh(status="Reset to the default location") + + return SettingsWidget + + +_settings_class = None + + +def settings_widget(): + """Return an interactive widget for editing em_database's settings (Jupyter). + + Backs ``display(em_database.settings)`` and ``em_database.settings.widget()``. + Requires anywidget (``pip install em-database[widget]``). + """ + _prepare_frontend() + global _settings_class + if _settings_class is None: + try: + _settings_class = _make_settings_class() + except ImportError as error: + raise ImportError( + "The settings widget needs anywidget. Install it with " + "`pip install em-database[widget]` (or `pip install anywidget`)." + ) from error + return _settings_class() + + def _make_toasts_class(): """Build the singleton ``DownloadToasts`` widget, importing anywidget lazily.""" import anywidget From 60787fe2513e4af61eed2a2e05b2e86caaae15be Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 20 Aug 2026 13:03:47 -0500 Subject: [PATCH 4/7] Settings widget: client-side folder Browse Pick a folder with a hidden webkitdirectory in the browser and fill the path field client-side (deriving the absolute path from File.path where the environment exposes it, e.g. desktop/Electron), then Save. No kernel-side dialog. --- em_database/static/browser.css | 1 + em_database/static/settings.js | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/em_database/static/browser.css b/em_database/static/browser.css index 818df0e..53b6db8 100644 --- a/em_database/static/browser.css +++ b/em_database/static/browser.css @@ -205,6 +205,7 @@ text-transform: uppercase; color: var(--emdb-muted); margin: 12px 0 6px; } .emdb-field-actions { display: flex; gap: 8px; margin-top: 8px; } +.emdb-hint { margin-top: 6px; font-size: 11px; color: var(--emdb-subtext); } .emdb-status { margin-top: 12px; font-size: 12px; color: var(--emdb-green); font-weight: 600; } .emdb-settings-info { margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--emdb-surface0); } .emdb-searchlist { display: flex; flex-direction: column; gap: 5px; } diff --git a/em_database/static/settings.js b/em_database/static/settings.js index 72e9767..618c0aa 100644 --- a/em_database/static/settings.js +++ b/em_database/static/settings.js @@ -41,7 +41,39 @@ function render({ model, el: root }) { input.spellcheck = false; field.appendChild(input); + // Client-side folder picker: a hidden directory . The browser only + // exposes the absolute path on desktop/Electron (File.path); a plain browser + // hides it, so we fall back to asking the user to type it. + const picker = el("input"); + picker.type = "file"; + picker.setAttribute("webkitdirectory", ""); + picker.setAttribute("directory", ""); + picker.style.display = "none"; + field.appendChild(picker); + const hint = el("div", "emdb-hint"); + field.appendChild(hint); + + picker.addEventListener("change", function () { + const files = picker.files; + if (files && files.length) { + const f = files[0]; + const rel = f.webkitRelativePath || f.name; + if (f.path) { // absolute path available (desktop/Electron) + input.value = f.path.slice(0, f.path.length - rel.length) + rel.split("/")[0]; + hint.textContent = "Selected " + input.value + " — click Save to keep it."; + } else { + hint.textContent = + "This browser can’t share the folder’s full path — type it above instead."; + } + } + picker.value = ""; // let the same folder be picked again + }); + const actions = el("div", "emdb-field-actions"); + const browse = el("button", "emdb-copy-btn", "Browse…"); + browse.title = "Pick a folder"; + browse.addEventListener("click", function () { picker.click(); }); + actions.appendChild(browse); const save = el("button", "emdb-dl", "Save"); save.title = "Set and remember across sessions"; save.addEventListener("click", function () { cmd("save", { data_dir: input.value.trim() }); }); From 2f35073782f99820e8b53cd26716e855ffce6907 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 20 Aug 2026 13:08:17 -0500 Subject: [PATCH 5/7] Settings widget: Browse to the right of the field, no "upload" prompt The webkitdirectory input made the browser show an "upload" dialog. Use a plain file instead (a normal "Open" dialog) and take the selected file's parent as the directory. Move the Browse button onto the directory line, to the right of the path field. --- em_database/static/browser.css | 9 +++++++++ em_database/static/settings.js | 37 +++++++++++++++++----------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/em_database/static/browser.css b/em_database/static/browser.css index 53b6db8..3ba9959 100644 --- a/em_database/static/browser.css +++ b/em_database/static/browser.css @@ -204,6 +204,15 @@ font-size: 10.5px; font-weight: 700; letter-spacing: 0.06em; text-transform: uppercase; color: var(--emdb-muted); margin: 12px 0 6px; } +.emdb-field-row { display: flex; gap: 8px; align-items: stretch; } +.emdb-field-row .emdb-search { flex: 1; min-width: 0; margin-top: 0; } +.emdb-browse { + flex: 0 0 auto; cursor: pointer; font-family: var(--emdb-font); + font-size: 12px; font-weight: 600; color: var(--emdb-text); + background: var(--emdb-surface0); border: 1px solid var(--emdb-surface1); + border-radius: 7px; padding: 0 14px; +} +.emdb-browse:hover { border-color: var(--emdb-blue); color: var(--emdb-blue); } .emdb-field-actions { display: flex; gap: 8px; margin-top: 8px; } .emdb-hint { margin-top: 6px; font-size: 11px; color: var(--emdb-subtext); } .emdb-status { margin-top: 12px; font-size: 12px; color: var(--emdb-green); font-weight: 600; } diff --git a/em_database/static/settings.js b/em_database/static/settings.js index 618c0aa..41bacd0 100644 --- a/em_database/static/settings.js +++ b/em_database/static/settings.js @@ -34,46 +34,47 @@ function render({ model, el: root }) { // Data directory field const field = el("div", "emdb-field"); field.appendChild(el("div", "emdb-field-label", "Data directory")); + // Input + Browse on one line (Browse to the right of the path field). + const row = el("div", "emdb-field-row"); const input = el("input", "emdb-search"); input.type = "text"; input.value = model.get("data_dir") || ""; input.placeholder = model.get("default_dir") || ""; input.spellcheck = false; - field.appendChild(input); + row.appendChild(input); + const browse = el("button", "emdb-browse", "Browse…"); + browse.title = "Pick any file inside the folder you want; its folder is used"; + row.appendChild(browse); + field.appendChild(row); - // Client-side folder picker: a hidden directory . The browser only - // exposes the absolute path on desktop/Electron (File.path); a plain browser - // hides it, so we fall back to asking the user to type it. + // Client-side picker: a plain file (a normal "Open" dialog, no + // "upload" prompt). We use the selected file's parent as the directory. The + // absolute path is available on desktop/Electron (File.path); a plain + // browser hides it, so we fall back to asking the user to type it. const picker = el("input"); picker.type = "file"; - picker.setAttribute("webkitdirectory", ""); - picker.setAttribute("directory", ""); picker.style.display = "none"; field.appendChild(picker); const hint = el("div", "emdb-hint"); field.appendChild(hint); + browse.addEventListener("click", function () { picker.click(); }); picker.addEventListener("change", function () { - const files = picker.files; - if (files && files.length) { - const f = files[0]; - const rel = f.webkitRelativePath || f.name; + const f = picker.files && picker.files[0]; + if (f) { if (f.path) { // absolute path available (desktop/Electron) - input.value = f.path.slice(0, f.path.length - rel.length) + rel.split("/")[0]; - hint.textContent = "Selected " + input.value + " — click Save to keep it."; + const dir = f.path.slice(0, f.path.length - f.name.length).replace(/[\\/]+$/, ""); + input.value = dir; + hint.textContent = "Selected " + dir + " — click Save to keep it."; } else { hint.textContent = - "This browser can’t share the folder’s full path — type it above instead."; + "This browser can’t share the folder’s path — type it above instead."; } } - picker.value = ""; // let the same folder be picked again + picker.value = ""; // let the same file be picked again }); const actions = el("div", "emdb-field-actions"); - const browse = el("button", "emdb-copy-btn", "Browse…"); - browse.title = "Pick a folder"; - browse.addEventListener("click", function () { picker.click(); }); - actions.appendChild(browse); const save = el("button", "emdb-dl", "Save"); save.title = "Set and remember across sessions"; save.addEventListener("click", function () { cmd("save", { data_dir: input.value.trim() }); }); From bd39c3c26ee61f5e8fbac49431afd56c0deeb2af Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Thu, 20 Aug 2026 13:34:42 -0500 Subject: [PATCH 6/7] Feature: redesign the docs site as a widget-style app Rebuild the documentation site to look and browse like the Jupyter widget (dark Catppuccin), with a top nav of Examples / API / All Data / Add Dataset. - Landing page: a prominent search bar + technique tabs over a live-filtered, grouped dataset list with a details panel - the same UI as em_database.browse(), fed by the real catalogue baked in at build time. - All Data: the full widget-style dataset browser. - Add Dataset: a schema-driven form (one control per json-schema field, enum dropdowns, repeatable authors, live YAML preview + client-side validation) that opens GitHub's prefilled create-file page to open a PR adding the dataset YAML - no backend needed on a static site. - Examples (sphinx-gallery) and API (autodoc) still build; a _static/custom.css themes the pydata pages to match the app. Generated pages are self-contained (inline browser.css + baked catalogue) and written by the existing build-finished hook. --- doc/source/_static/custom.css | 141 ++++++++ doc/source/add_dataset.rst | 15 + doc/source/all_data.rst | 12 + doc/source/conf.py | 70 +++- doc/source/index.rst | 23 +- em_database/_build_docs.py | 628 +++++++++++++++++++++++++++++++--- 6 files changed, 832 insertions(+), 57 deletions(-) create mode 100644 doc/source/_static/custom.css create mode 100644 doc/source/add_dataset.rst create mode 100644 doc/source/all_data.rst diff --git a/doc/source/_static/custom.css b/doc/source/_static/custom.css new file mode 100644 index 0000000..4290a9b --- /dev/null +++ b/doc/source/_static/custom.css @@ -0,0 +1,141 @@ +/* custom.css - Catppuccin Mocha theming for the Sphinx (pydata) pages + * + * The landing / All Data / Add Dataset pages are generated as self-contained + * app pages (see em_database/_build_docs.py). The Examples gallery and API + * reference are rendered by pydata_sphinx_theme; this file recolors that theme + * to match, so the whole site reads as one dark Catppuccin "app". It maps the + * Catppuccin palette onto pydata's --pst-color-* variables and restyles the + * top navbar to echo the app header. + */ + +:root { + --ctp-base: #1e1e2e; + --ctp-mantle: #181825; + --ctp-crust: #11111b; + --ctp-surface0: #313244; + --ctp-surface1: #45475a; + --ctp-surface2: #585b70; + --ctp-overlay: #2a2a3c; + --ctp-text: #cdd6f4; + --ctp-subtext: #a6adc8; + --ctp-muted: #7f849c; + --ctp-blue: #89b4fa; + --ctp-mauve: #cba6f7; + --ctp-green: #a6e3a1; + --ctp-red: #f38ba8; +} + +/* Map Catppuccin onto pydata's tokens. Apply to the dark theme (the default), + * and also to the base :root so the palette holds before the theme attribute is + * stamped on / if the visitor forces light. */ +:root, +html[data-theme="dark"], +html[data-theme="light"] { + --pst-color-primary: var(--ctp-blue); + --pst-color-primary-highlight: var(--ctp-mauve); + --pst-color-secondary: var(--ctp-mauve); + --pst-color-secondary-highlight: var(--ctp-blue); + --pst-color-background: var(--ctp-base); + --pst-color-on-background: var(--ctp-mantle); + --pst-color-surface: var(--ctp-surface0); + --pst-color-on-surface: var(--ctp-text); + --pst-color-text-base: var(--ctp-text); + --pst-color-text-muted: var(--ctp-subtext); + --pst-color-heading-color: var(--ctp-text); + --pst-color-border: var(--ctp-surface0); + --pst-color-shadow: rgba(0, 0, 0, 0.45); + --pst-color-link: var(--ctp-blue); + --pst-color-link-hover: var(--ctp-mauve); + --pst-color-inline-code: var(--ctp-mauve); + --pst-color-inline-code-links: var(--ctp-blue); + --pst-color-target: rgba(137, 180, 250, 0.18); + --pst-color-accent: var(--ctp-mauve); + --pst-color-info: var(--ctp-blue); + --pst-color-success: var(--ctp-green); + --pst-color-warning: var(--ctp-yellow, #f9e2af); + --pst-color-danger: var(--ctp-red); + color-scheme: dark; +} + +body { + background-color: var(--ctp-base); + color: var(--ctp-text); +} + +/* Top navbar: echo the app header gradient + pill links. */ +.bd-header, +.bd-header .navbar-header-items, +.navbar { + background: linear-gradient(#1c1c2b, var(--ctp-mantle)) !important; + border-bottom: 1px solid var(--ctp-surface0); + box-shadow: 0 2px 16px rgba(0, 0, 0, 0.35); +} +.bd-header .navbar-brand, +.navbar-brand p, +.navbar-brand .title { + color: var(--ctp-text) !important; + font-weight: 800; +} +.bd-header .nav-link, +.bd-header .navbar-nav .nav-link { + color: var(--ctp-subtext) !important; + font-weight: 600; + border-radius: 999px; + padding: 6px 13px !important; + border: 1px solid transparent; +} +.bd-header .nav-link:hover, +.bd-header .navbar-nav .nav-link:hover { + color: var(--ctp-text) !important; + background: var(--ctp-overlay); +} +.bd-header .navbar-nav .current > .nav-link, +.bd-header .nav-link.active { + color: var(--ctp-blue) !important; + background: rgba(137, 180, 250, 0.14); + border-color: rgba(137, 180, 250, 0.4); +} + +/* Sidebars + content surfaces. */ +.bd-sidebar-primary, +.bd-sidebar-secondary { + background-color: var(--ctp-mantle); + border-color: var(--ctp-surface0); +} +.bd-main .bd-content { color: var(--ctp-text); } + +/* Gradient page titles, matching the app hero. */ +.bd-article-container h1 { + background: linear-gradient(135deg, #89b4fa, #cba6f7); + -webkit-background-clip: text; + background-clip: text; + -webkit-text-fill-color: transparent; + font-weight: 800; +} + +/* Code + admonitions. */ +code, pre, .highlight { + background-color: var(--ctp-crust) !important; +} +.highlight pre { color: var(--ctp-text); } + +/* Cards (sphinx-design) + gallery thumbnails. */ +.sd-card, +.sphx-glr-thumbcontainer { + background-color: var(--ctp-mantle) !important; + border: 1px solid var(--ctp-surface0) !important; + border-radius: 10px; +} +.sd-card:hover, +.sphx-glr-thumbcontainer:hover { + border-color: var(--ctp-blue) !important; + box-shadow: 0 8px 30px rgba(0, 0, 0, 0.35); +} + +/* Buttons / search. */ +.bd-search input, +input[type="search"] { + background-color: var(--ctp-crust); + color: var(--ctp-text); + border: 1px solid var(--ctp-surface0); +} diff --git a/doc/source/add_dataset.rst b/doc/source/add_dataset.rst new file mode 100644 index 0000000..a52acd5 --- /dev/null +++ b/doc/source/add_dataset.rst @@ -0,0 +1,15 @@ +:html_theme.sidebar_secondary.remove: + +Add Dataset +=========== + +A submission form that opens a pre-filled GitHub pull request adding your +dataset's YAML file. + +.. note:: + + This page is rendered as a self-contained form during the build (see + ``build-finished`` in ``conf.py``); this text is a fallback that only shows if + that generation step fails. You can also submit a dataset via the + `new-dataset issue form + `_. diff --git a/doc/source/all_data.rst b/doc/source/all_data.rst new file mode 100644 index 0000000..3e5dabb --- /dev/null +++ b/doc/source/all_data.rst @@ -0,0 +1,12 @@ +:html_theme.sidebar_secondary.remove: + +All Data +======== + +The full, widget-styled dataset browser. + +.. note:: + + This page is rendered as a self-contained, widget-styled browser during the + build (see ``build-finished`` in ``conf.py``); this text is a fallback that + only shows if that generation step fails. diff --git a/doc/source/conf.py b/doc/source/conf.py index 0a0984a..4b326e3 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -6,7 +6,14 @@ import sys from pathlib import Path # Import and run the build script -from em_database._build_docs import parse_datasets, generate_html_table, generate_browser_html +from em_database._build_docs import ( + parse_datasets, + generate_html_table, + generate_browser_html, + generate_landing_html, + generate_all_data_html, + generate_add_dataset_html, +) # Add project root to path sys.path.insert(0, str(Path(__file__).parent.parent)) @@ -32,7 +39,10 @@ ] templates_path = ['_templates'] -exclude_patterns = [] +# intro.rst / datasets.rst are superseded by the generated landing + All Data +# app pages; keep the files but leave them out of the build so they don't warn +# about being orphaned. +exclude_patterns = ['intro.rst', 'datasets.rst'] @@ -41,9 +51,39 @@ html_theme = "pydata_sphinx_theme" html_static_path = ["_static"] +html_css_files = ["custom.css"] master_doc = "index" + +# Dark Catppuccin-Mocha by default, to match the generated app pages. +html_context = {"default_mode": "dark"} + +# Top navigation: Examples / API / All Data / Add Dataset. Examples and API are +# Sphinx-generated (sphinx-gallery + autodoc); All Data and Add Dataset are the +# generated app pages, in the toctree so pydata builds correct relative links to +# them from every page (their HTML output is then overwritten with the app page +# in the build-finished hook below). +html_theme_options = { + "logo": {"text": "◆ EM-Database"}, + "navbar_start": ["navbar-logo"], + "navbar_center": ["navbar-nav"], + "navbar_end": ["theme-switcher", "navbar-icon-links"], + "navbar_persistent": [], + "show_prev_next": False, + "icon_links": [ + { + "name": "GitHub", + "url": "https://github.com/CSSFrancis/em_data", + "icon": "fa-brands fa-github", + }, + ], +} + +# No sidebars on the app-style pages (their output is fully replaced anyway). html_sidebars = { - "datasets": [] + "index": [], + "all_data": [], + "add_dataset": [], + "datasets": [], } def build_datasets_html(app, exception): @@ -65,13 +105,23 @@ def build_datasets_html(app, exception): with output_path.open('w', encoding='utf-8') as f: f.write(html_output) - # The widget-styled browser used on the landing page. - try: - browser_html = generate_browser_html() - (Path(app.outdir) / 'datasets_browser.html').write_text(browser_html, encoding='utf-8') - print("Wrote datasets_browser.html") - except Exception as e: # pragma: no cover - keep the build alive - print(f"Could not build datasets_browser.html: {e}") + # Generated, self-contained Catppuccin "app" pages. Each is written into the + # build output (overwriting the Sphinx-rendered page where names collide: + # index.html, all_data.html, add_dataset.html) so the whole site looks like + # em_database.browse(). Each is guarded so a failure never kills the build. + outdir = Path(app.outdir) + pages = { + 'index.html': generate_landing_html, + 'all_data.html': generate_all_data_html, + 'add_dataset.html': generate_add_dataset_html, + 'datasets_browser.html': generate_browser_html, + } + for filename, generator in pages.items(): + try: + (outdir / filename).write_text(generator(), encoding='utf-8') + print(f"Wrote {filename}") + except Exception as e: # pragma: no cover - keep the build alive + print(f"Could not build {filename}: {e}") def setup(app): app.connect('build-finished', build_datasets_html) diff --git a/doc/source/index.rst b/doc/source/index.rst index 5235120..919e505 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,10 +1,23 @@ +:html_theme.sidebar_secondary.remove: + .. toctree:: - :maxdepth: 1 :hidden: - Introduction - Reference - Datasets Examples + API + All Data + Add Dataset + +EM-Database +=========== + +A curated, citable collection of electron microscopy datasets — a couple of +lines of Python from your analysis. + +.. note:: -.. include:: intro.rst \ No newline at end of file + This landing page is rendered as a self-contained, widget-styled app during + the build (see ``build-finished`` in ``conf.py``); this reStructuredText is a + fallback that only shows if that generation step fails. Browse the datasets on + the :doc:`All Data ` page, or add one on :doc:`Add Dataset + `. diff --git a/em_database/_build_docs.py b/em_database/_build_docs.py index 5f8a532..fa312ca 100644 --- a/em_database/_build_docs.py +++ b/em_database/_build_docs.py @@ -1,3 +1,4 @@ +import json import yaml from pathlib import Path from collections import defaultdict @@ -451,7 +452,8 @@ def generate_html_table(datasets_by_technique): } function drawTabs() { tabsEl.innerHTML = ""; - ["All"].concat(techniques()).forEach(function (tab) { + var tabList = (typeof TABS !== "undefined" && TABS) ? TABS : techniques(); + ["All"].concat(tabList).forEach(function (tab) { var label = tab === "All" ? "All" : (TAB_LABEL[tab] || tab); var b = el("button", "emdb-tab" + (state.tab === tab ? " active" : ""), esc(label)); b.addEventListener("click", function () { state.tab = tab; drawTabs(); drawList(); }); @@ -527,60 +529,602 @@ def generate_html_table(datasets_by_technique): })(); """ -_BROWSER_PAGE = """ - - -EM Datasets - - -
- - """ +# Form styling for the Add Dataset page. +_FORM_CSS = """ +.form-wrap { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 430px); gap: 26px; align-items: start; } +@media (max-width: 900px) { .form-wrap { grid-template-columns: 1fr; } .yaml-side { position: static; } } +.ds-form { display: flex; flex-direction: column; gap: 14px; } +.grid2 { display: grid; grid-template-columns: 1fr 1fr; gap: 14px; } +@media (max-width: 560px) { .grid2 { grid-template-columns: 1fr; } } +.field { display: flex; flex-direction: column; gap: 4px; min-width: 0; } +.field label { font-size: 11px; font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: var(--emdb-muted); } +.field .req { color: var(--emdb-red); } +.field-hint { font-size: 11px; color: var(--emdb-subtext); } +.field input, .field textarea, .field select { + width: 100%; box-sizing: border-box; + background: var(--emdb-crust); color: var(--emdb-text); + border: 1px solid var(--emdb-surface0); border-radius: 7px; padding: 8px 10px; + font-size: 13px; font-family: var(--emdb-font); outline: none; +} +.field textarea { min-height: 84px; resize: vertical; line-height: 1.5; } +.field input:focus, .field textarea:focus, .field select:focus { border-color: var(--emdb-blue); } +.field input.invalid, .field textarea.invalid, .field select.invalid { border-color: var(--emdb-red); } +.field-err { font-size: 11px; color: var(--emdb-red); min-height: 13px; } +.section-title { + font-size: 12px; font-weight: 800; letter-spacing: 0.06em; text-transform: uppercase; + color: var(--emdb-subtext); margin-top: 8px; padding-top: 16px; border-top: 1px solid var(--emdb-surface0); +} +#authors { display: flex; flex-direction: column; gap: 12px; } +.author-row { + display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 10px; + background: var(--emdb-mantle); border: 1px solid var(--emdb-surface0); border-radius: 9px; padding: 12px; +} +@media (max-width: 620px) { .author-row { grid-template-columns: 1fr; } } +.btn-ghost { + align-self: flex-start; cursor: pointer; font-weight: 600; font-size: 12px; + background: var(--emdb-surface0); color: var(--emdb-text); + border: 1px solid var(--emdb-surface1); border-radius: 7px; padding: 6px 12px; +} +.btn-ghost:hover { border-color: var(--emdb-blue); color: var(--emdb-blue); } + +.yaml-side { position: sticky; top: 72px; display: flex; flex-direction: column; gap: 12px; } +.yaml-head { + display: flex; align-items: center; justify-content: space-between; + font-size: 11px; font-weight: 800; letter-spacing: 0.06em; text-transform: uppercase; color: var(--emdb-muted); +} +.yaml-pre { + margin: 0; background: var(--emdb-crust); border: 1px solid var(--emdb-surface0); + border-radius: 10px; padding: 14px; max-height: 440px; overflow: auto; + font-family: var(--emdb-mono); font-size: 12px; color: var(--emdb-text); white-space: pre; +} +.submit-row { display: flex; flex-direction: column; gap: 10px; } +.btn-primary { + cursor: pointer; font-weight: 700; font-size: 14px; text-align: center; + color: var(--emdb-crust); background: linear-gradient(135deg, var(--emdb-blue), var(--emdb-mauve)); + border: 0; border-radius: 9px; padding: 12px 16px; +} +.btn-primary:disabled { opacity: 0.45; cursor: not-allowed; filter: grayscale(0.3); } +.btn-primary:not(:disabled):hover { filter: brightness(1.06); } +.btn-secondary { + text-align: center; font-size: 12px; font-weight: 600; color: var(--emdb-subtext); + border: 1px solid var(--emdb-surface1); border-radius: 8px; padding: 9px 14px; +} +.btn-secondary:hover { color: var(--emdb-text); border-color: var(--emdb-blue); text-decoration: none; } +.copy-mini { + cursor: pointer; font-size: 11px; font-weight: 600; color: var(--emdb-blue); + background: transparent; border: 1px solid var(--emdb-surface1); border-radius: 6px; padding: 2px 9px; +} +.copy-mini:hover { border-color: var(--emdb-blue); background: rgba(137, 180, 250, 0.1); } +.copy-mini.copied { color: var(--emdb-green); border-color: var(--emdb-green); } +.form-note { font-size: 11.5px; color: var(--emdb-subtext); line-height: 1.55; } +.form-note code { font-family: var(--emdb-mono); color: var(--emdb-text); } +""" + + +def _esc(value) -> str: + """Minimal HTML escaping for text baked into a page at build time.""" + return (str(value).replace("&", "&").replace("<", "<").replace(">", ">") + .replace('"', """).replace("'", "'")) -def generate_browser_html(): - """Generate a self-contained, widget-styled dataset browser for the docs. - Reuses ``em_database/static/browser.css`` and the ``em_database.catalogue`` - data model, so the docs landing page looks and browses like - ``em_database.browse()`` - minus the live downloads (a static site has no - kernel): the details panel offers the copy-to-load snippet and a direct - download link instead. +# Top-nav destinations. Examples and API are Sphinx-generated (sphinx-gallery + +# autodoc); All Data and Add Dataset are the generated app pages. All the +# generated pages sit at the site root, so these relative links resolve the +# same from each of them. +_NAV_LINKS = ( + ("Examples", "examples/index.html"), + ("API", "reference/index.html"), + ("All Data", "all_data.html"), + ("Add Dataset", "add_dataset.html"), +) + + +def _top_nav(active: str = "") -> str: + items = "".join( + '{name}'.format( + cls=" active" if name == active else "", url=url, name=_esc(name)) + for name, url in _NAV_LINKS + ) + return ( + '' + ) + + +def _load_css() -> str: + return (Path(__file__).parent / "static" / "browser.css").read_text(encoding="utf-8") + + +def _app_page(title: str, body: str, active: str = "", extra_css: str = "", + scripts: str = "") -> str: + """Wrap page ``body`` in the self-contained Catppuccin app shell. + + Built by concatenation (not ``str.format``/``%``) so CSS/JS braces need no + escaping. The result references no external hosts - all CSS/JS is inline. """ - import json + return ( + "\n" + '\n' + '\n' + "" + _esc(title) + "\n" + "\n" + "\n\n" + + _top_nav(active) + "\n" + body + "\n" + scripts + "\n" + "\n" + ) + + +def _catalogue_payload(): + """``(payload, tabs)`` - the baked catalogue and the ordered tab list. + Tabs are the canonical techniques (so 4D-STEM / EELS / EDS / EBSD / STEM / + In-situ / Cryo always show, even if a technique currently has no dataset), + followed by any other technique that happens to be present. + """ from em_database import catalogue payload = catalogue.catalogue() - css = (Path(__file__).parent / "static" / "browser.css").read_text(encoding="utf-8") - page = _BROWSER_PAGE.replace("__CSS__", css) - page = page.replace("__DATA__", json.dumps(payload)) - page = page.replace("__JS__", _DOCS_BROWSER_JS) - return page + order = list(catalogue.TECHNIQUE_ORDER) + present = [g["technique"] for g in payload.get("groups", [])] + tabs = order + [t for t in present if t not in order] + return payload, tabs -if __name__ == "__main__": +def _browser_script(payload, tabs) -> str: + """The " + ) + + +def generate_browser_html() -> str: + """Self-contained, widget-styled dataset browser (no nav chrome). + + Kept for backward compatibility (it can be embedded in an ``