diff --git a/README.md b/README.md
index 7879fef..9fe94d1 100644
--- a/README.md
+++ b/README.md
@@ -35,17 +35,49 @@ Call `path.done()` to check progress without blocking, or `path.result()` to wai
explicitly. Pass `download(background=False)` to block and get the path as a plain
string instead.
-The download location defaults to `~/em_database` and can be changed either for the
-session or through the `EM_DATABASE_DATA_DIR` environment variable:
+## Settings
+
+Configuration lives in a live, matplotlib-`rcParams`-style object,
+`em_database.settings`, seeded at import from `~/.em_database/settings.yaml`.
+Change it in memory for an immediate effect, and persist it to remember the
+choice across sessions:
```python
import em_database
-em_database.set_data_dir("/path/to/somewhere")
-em_database.get_data_dir()
-em_database.reset_data_dir()
+em_database.settings["data_dir"] = "/big/disk/em_data" # takes effect now
+em_database.settings.save() # remember it next time
```
+In Jupyter you can also edit them interactively — `display(em_database.settings)`
+renders a panel to set (and save) the data directory directly.
+
+The data directory defaults to `~/em_database`. Convenience helpers wrap the
+common case — `set_data_dir` persists by default:
+
+```python
+em_database.get_data_dir() # current location
+em_database.set_data_dir("/big/disk/em_data") # set + persist
+em_database.set_data_dir("/scratch", persist=False) # one-off, in-memory only
+em_database.reset_data_dir() # back to the default, forget the choice
+```
+
+No environment variable is needed. Only values you explicitly set are written to
+the file, so the default is never frozen in — it keeps being obeyed even if it
+changes. For a one-off override (e.g. CI) the legacy `EM_DATABASE_DATA_DIR` is
+still honored, and `EM_DATABASE_CONFIG` relocates the settings file.
+
+### Shared, system-wide data
+
+Datasets can be installed once for every user. `download()` and `filepath()`
+look in the shared/system locations **first**, then your own data directory, and
+only download (into your directory) if the file is nowhere to be found — so a
+shared copy is reused instead of refetched. Shared locations come from the
+`EM_DATABASE_SHARED_DIR` environment variable (an `os.pathsep`-separated list), a
+`shared_data_dirs` list in your settings, or a system config file
+(`/etc/em_database/settings.yaml`, or `%PROGRAMDATA%\em_database\settings.yaml`
+on Windows).
+
## Adding a dataset
Datasets are described by a YAML file in `em_database/datasets/`, one entry per file,
diff --git a/em_database/__init__.py b/em_database/__init__.py
index c9f9cc2..adacee5 100644
--- a/em_database/__init__.py
+++ b/em_database/__init__.py
@@ -1,19 +1,12 @@
### Example datasets ###
import os
+from em_database.config import settings
from em_database.downloadable_dataset import DownloadableDataset
from em_database._create_stubs import build_docstring
from em_database import data
__all__ = []
-if "EM_DATABASE_DATA_DIR" not in os.environ:
- # set the default dir to User's home directory + "/emdata"
- os.environ["EM_DATABASE_DATA_DIR"] = os.path.join(
- os.path.expanduser("~"),"em_database"
- )
-
-
-
def get_data_dir():
"""
Get the directory where example datasets are stored.
@@ -23,9 +16,10 @@ def get_data_dir():
str
Path to the example datasets directory.
"""
- return os.environ["EM_DATABASE_DATA_DIR"]
+ from em_database import config
+ return config.data_dir()
-def set_data_dir(path: str):
+def set_data_dir(path: str, persist: bool = True):
"""
Set the directory where example datasets are stored.
@@ -33,16 +27,31 @@ def set_data_dir(path: str):
----------
path : str
Path to the desired example datasets directory.
+ persist : bool, optional
+ If True (the default), remember the choice across sessions by writing it
+ to the settings file. Pass False for a one-off, in-memory change.
"""
- os.environ["EM_DATABASE_DATA_DIR"] = path
+ settings["data_dir"] = str(path)
+ if persist:
+ settings.save()
def reset_data_dir():
"""
- Reset the example datasets directory to the default location.
+ Reset the example datasets directory to the default location, clearing any
+ saved choice.
"""
- os.environ["EM_DATABASE_DATA_DIR"] = os.path.join(
- os.path.expanduser("~"),"em_database"
- )
+ settings.reset("data_dir")
+
+
+def get_setting(key: str, default=None):
+ """Read a value from :data:`em_database.settings`."""
+ return settings.get(key, default)
+
+def set_setting(key: str, value, persist: bool = True):
+ """Set a value in :data:`em_database.settings`, persisting it by default."""
+ settings[key] = value
+ if persist:
+ settings.save()
def browse(**kwargs):
@@ -60,7 +69,8 @@ def browse(**kwargs):
return _browse(**kwargs)
-__all__ = ['get_data_dir', 'set_data_dir', 'reset_data_dir', 'browse', "data"]
+__all__ = ['get_data_dir', 'set_data_dir', 'reset_data_dir',
+ 'get_setting', 'set_setting', 'settings', 'browse', "data"]
# Let ``display(em_database)`` render the browser. Reassigning the module's
diff --git a/em_database/config.py b/em_database/config.py
new file mode 100644
index 0000000..1fa7bb3
--- /dev/null
+++ b/em_database/config.py
@@ -0,0 +1,213 @@
+"""Runtime settings for em_database, in the style of matplotlib's ``rcParams``.
+
+``em_database.settings`` is a live dict, seeded at import from a YAML file (or
+the built-in defaults). Change it in memory::
+
+ import em_database
+ em_database.settings["data_dir"] = "/big/disk/em_data"
+
+and it takes effect immediately. To remember the choice across sessions, persist
+it::
+
+ em_database.settings.save()
+
+Settings live in a ``~/.em_database`` folder (``~/.em_database/settings.yaml``),
+or wherever ``EM_DATABASE_CONFIG`` points. No environment variable is *needed* to
+configure em_database - the settings object is the source of truth; the old
+``EM_DATABASE_DATA_DIR`` is still honored as a seed for backward compatibility.
+
+Data can also be installed **system-wide** and shared by every user. When
+resolving where a dataset lives, em_database looks in the shared/system
+locations first, then the user's own data directory, and only downloads (into
+the user directory) if it is nowhere to be found. Shared locations come from a
+system config file (see :func:`system_config_path`), the ``EM_DATABASE_SHARED_DIR``
+environment variable (an ``os.pathsep``-separated list), or a ``shared_data_dirs``
+list in the user's settings.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import yaml
+
+
+def _default_data_dir() -> str:
+ return os.path.join(os.path.expanduser("~"), "em_database")
+
+
+#: Built-in defaults - the fallback when nothing is configured.
+DEFAULTS: dict = {
+ "data_dir": _default_data_dir(),
+}
+
+
+def config_dir() -> Path:
+ """The ``~/.em_database`` folder that holds the settings file."""
+ return Path(os.path.expanduser("~")) / ".em_database"
+
+
+def config_path() -> Path:
+ """Path to the YAML settings file.
+
+ ``EM_DATABASE_CONFIG`` overrides the location; otherwise it is
+ ``~/.em_database/settings.yaml``.
+ """
+ override = os.environ.get("EM_DATABASE_CONFIG")
+ if override:
+ return Path(override)
+ return config_dir() / "settings.yaml"
+
+
+def system_config_path() -> Path | None:
+ """Path to the machine-wide settings file (for system-wide installs).
+
+ ``EM_DATABASE_SYSTEM_CONFIG`` overrides it; otherwise it is
+ ``/etc/em_database/settings.yaml`` (POSIX) or
+ ``%PROGRAMDATA%\\em_database\\settings.yaml`` (Windows).
+ """
+ override = os.environ.get("EM_DATABASE_SYSTEM_CONFIG")
+ if override:
+ return Path(override)
+ if os.name == "nt":
+ base = os.environ.get("PROGRAMDATA")
+ return Path(base) / "em_database" / "settings.yaml" if base else None
+ return Path("/etc/em_database/settings.yaml")
+
+
+def _read_yaml(path: Path | None) -> dict:
+ """Parse a YAML file into a dict, or ``{}`` if missing/unreadable/malformed."""
+ if path is None:
+ return {}
+ try:
+ with open(path, encoding="utf-8") as handle:
+ data = yaml.safe_load(handle)
+ return data if isinstance(data, dict) else {}
+ except FileNotFoundError:
+ return {}
+ except Exception:
+ # A malformed settings file must not break importing the package.
+ return {}
+
+
+def _read_file() -> dict:
+ """The user settings stored on disk, or ``{}``."""
+ return _read_yaml(config_path())
+
+
+def _read_system() -> dict:
+ """The machine-wide settings, or ``{}``."""
+ return _read_yaml(system_config_path())
+
+
+def _write_file(data: dict) -> None:
+ path = config_path()
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with open(path, "w", encoding="utf-8") as handle:
+ yaml.safe_dump(dict(data), handle, default_flow_style=False, sort_keys=True)
+
+
+def _seed(settings: "Settings") -> None:
+ """(Re)seed a settings object: defaults < file < legacy env var."""
+ settings.clear()
+ settings.update(DEFAULTS)
+ settings.update(_read_file()) # the file overrides the defaults
+ # Backward compatibility: seed from the old env var if it is set.
+ env_dir = os.environ.get("EM_DATABASE_DATA_DIR")
+ if env_dir:
+ settings["data_dir"] = env_dir
+
+
+class Settings(dict):
+ """A live settings mapping that can persist itself to the YAML file."""
+
+ def save(self) -> None:
+ """Persist the settings to the YAML file.
+
+ Only values that DIFFER from the built-in defaults are written, so a
+ default (like the data directory) is never frozen into the file: it stays
+ dynamic and keeps being obeyed across sessions, machines, or a change to
+ the default itself. When nothing differs, any stale file is removed.
+ """
+ delta = {key: value for key, value in self.items() if DEFAULTS.get(key) != value}
+ path = config_path()
+ if delta:
+ _write_file(delta)
+ elif path.exists():
+ path.unlink()
+
+ def reload(self) -> None:
+ """Re-seed from the built-in defaults and the on-disk file, discarding
+ any in-memory changes."""
+ _seed(self)
+
+ def reset(self, key: str | None = None) -> None:
+ """Reset a key (or everything) to its default, in memory and on disk."""
+ if key is None:
+ self.clear()
+ self.update(DEFAULTS)
+ elif key in DEFAULTS:
+ self[key] = DEFAULTS[key]
+ else:
+ self.pop(key, None)
+ self.save() # delta-save drops the now-default key from the file
+
+ def widget(self):
+ """Return an interactive widget for editing the settings (Jupyter)."""
+ from em_database.widget import settings_widget
+ return settings_widget()
+
+ def _repr_mimebundle_(self, **kwargs):
+ """Render as an editable settings widget in Jupyter (falls back to the
+ plain dict repr if anywidget is not installed)."""
+ try:
+ from em_database.widget import settings_widget
+ widget = settings_widget()
+ except Exception:
+ return {"text/plain": repr(dict(self))}
+ return widget._repr_mimebundle_(**kwargs)
+
+
+#: The live settings object (seeded once at import).
+settings = Settings()
+_seed(settings)
+
+
+def data_dir() -> str:
+ """The user's data directory - where downloads are written."""
+ return str(settings.get("data_dir") or _default_data_dir())
+
+
+def shared_data_dirs() -> list[str]:
+ """System-wide / shared data locations, checked before the user's dir.
+
+ Order: ``EM_DATABASE_SHARED_DIR`` (an ``os.pathsep`` list), then the user's
+ ``shared_data_dirs`` setting, then the system config file's ``data_dir`` and
+ ``shared_data_dirs``. Duplicates are removed, order preserved.
+ """
+ dirs: list[str] = []
+ env = os.environ.get("EM_DATABASE_SHARED_DIR")
+ if env:
+ dirs += [d for d in env.split(os.pathsep) if d]
+ dirs += [str(d) for d in (settings.get("shared_data_dirs") or [])]
+ system = _read_system()
+ if system.get("data_dir"):
+ dirs.append(str(system["data_dir"]))
+ dirs += [str(d) for d in (system.get("shared_data_dirs") or [])]
+ seen: set[str] = set()
+ unique: list[str] = []
+ for d in dirs:
+ if d not in seen:
+ seen.add(d)
+ unique.append(d)
+ return unique
+
+
+def data_search_dirs() -> list[str]:
+ """Everywhere to look for an existing dataset: shared/system dirs first,
+ then the user's data directory."""
+ dirs = shared_data_dirs()
+ user = data_dir()
+ if user not in dirs:
+ dirs.append(user)
+ return dirs
diff --git a/em_database/downloadable_dataset.py b/em_database/downloadable_dataset.py
index e07c9f0..ad7cfbe 100644
--- a/em_database/downloadable_dataset.py
+++ b/em_database/downloadable_dataset.py
@@ -129,8 +129,8 @@ def _repr_mimebundle_(self, **kwargs):
def _resolve_destination(destination: str | None) -> str:
"""Return the directory the dataset should live in."""
if destination is None:
- return os.environ.get("EM_DATABASE_DATA_DIR",
- os.path.join(os.path.expanduser("~"), "em_database"))
+ from em_database import config
+ return config.data_dir()
return destination
def download(self,
@@ -176,7 +176,12 @@ def download(self,
"""
if not background:
return self._retrieve(destination, progressbar, chunk_size)
- target = os.path.join(self._resolve_destination(destination), self.file)
+ # Resolve where the file will end up: an existing shared/user copy if it
+ # is already present, otherwise the user's download location.
+ if destination is not None:
+ target = os.path.join(destination, self.file)
+ else:
+ target = self.filepath() or os.path.join(self._resolve_destination(None), self.file)
# In Jupyter (with the widget installed) a background download pops a
# cancelable toast; the toast's monitor replaces the plain progress bar.
monitor = finish = None
@@ -198,14 +203,25 @@ def _retrieve(self,
destination: str | None = None,
progressbar: bool = True,
chunk_size: int = 4096) -> str:
- """Fetch the file with pooch and return its local path (blocking)."""
+ """Fetch the file and return its local path (blocking).
+
+ With no explicit destination, an existing system-wide/shared copy is used
+ as-is (never re-downloaded); otherwise pooch downloads into the user's
+ data directory.
+ """
if progressbar:
try:
import tqdm # noqa: F401
except ImportError:
print("`tqdm` is not installed, progress bar will be disabled.")
progressbar = False
- destination = self._resolve_destination(destination)
+ if destination is None:
+ shared = self._find_shared()
+ if shared is not None:
+ return shared
+ destination = self._resolve_destination(None)
+ else:
+ destination = self._resolve_destination(destination)
# Instantiate an Http downloader with a custom user agent
headers = {"User-Agent": "em_database (https://github.com/CSSFrancis/em_data)"}
downloader = pooch.HTTPDownloader(progressbar=progressbar,
@@ -220,17 +236,26 @@ def _retrieve(self,
)
return filepath
+ def _find_shared(self) -> str | None:
+ """Path to an existing copy in a shared/system data dir, or None."""
+ from em_database import config
+ for directory in config.shared_data_dirs():
+ candidate = os.path.join(directory, self.file)
+ if os.path.exists(candidate):
+ return candidate
+ return None
+
def filepath(self) -> str:
- """ Return the local file path of the dataset if downloaded.
-
- If not downloaded return None. """
- destination = os.environ.get("EM_DATABASE_DATA_DIR",
- os.path.join(os.path.expanduser("~"), "em_database"))
- filepath = os.path.join(destination, self.file)
- if os.path.exists(filepath):
- return filepath
- else:
- return None
+ """ Return the local file path of the dataset if present.
+
+ Looks in the shared/system data locations first, then the user's data
+ directory. Returns None if the dataset is not downloaded anywhere. """
+ from em_database import config
+ for directory in config.data_search_dirs():
+ candidate = os.path.join(directory, self.file)
+ if os.path.exists(candidate):
+ return candidate
+ return None
def delete(self, destination: str | None = None) -> bool:
""" Delete the downloaded file if it is present.
diff --git a/em_database/static/browser.css b/em_database/static/browser.css
index b4eb66d..3ba9959 100644
--- a/em_database/static/browser.css
+++ b/em_database/static/browser.css
@@ -196,6 +196,39 @@
.emdb-inline-toast.cancelling { opacity: 0.65; }
.emdb-inline-toast.error { border-color: var(--emdb-red); }
+/* Settings panel (display(em_database.settings)) ------------------- */
+.emdb-settings-host { max-width: 620px; }
+.emdb-settings { padding: 16px 18px; }
+.emdb-field { margin-top: 12px; }
+.emdb-field-label {
+ font-size: 10.5px; font-weight: 700; letter-spacing: 0.06em;
+ text-transform: uppercase; color: var(--emdb-muted); margin: 12px 0 6px;
+}
+.emdb-field-row { display: flex; gap: 8px; align-items: stretch; }
+.emdb-field-row .emdb-search { flex: 1; min-width: 0; margin-top: 0; }
+.emdb-browse {
+ flex: 0 0 auto; cursor: pointer; font-family: var(--emdb-font);
+ font-size: 12px; font-weight: 600; color: var(--emdb-text);
+ background: var(--emdb-surface0); border: 1px solid var(--emdb-surface1);
+ border-radius: 7px; padding: 0 14px;
+}
+.emdb-browse:hover { border-color: var(--emdb-blue); color: var(--emdb-blue); }
+.emdb-field-actions { display: flex; gap: 8px; margin-top: 8px; }
+.emdb-hint { margin-top: 6px; font-size: 11px; color: var(--emdb-subtext); }
+.emdb-status { margin-top: 12px; font-size: 12px; color: var(--emdb-green); font-weight: 600; }
+.emdb-settings-info { margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--emdb-surface0); }
+.emdb-searchlist { display: flex; flex-direction: column; gap: 5px; }
+.emdb-searchdir {
+ font-family: var(--emdb-mono); font-size: 11.5px; color: var(--emdb-text);
+ word-break: break-all;
+}
+.emdb-tag {
+ font-family: var(--emdb-font); font-size: 10px; font-weight: 600;
+ color: var(--emdb-crust); background: var(--emdb-green);
+ border-radius: 4px; padding: 1px 6px; white-space: nowrap;
+}
+.emdb-tag.shared { background: var(--emdb-blue); }
+
/* Toasts: pinned to the bottom-right of the viewport ----------------- */
.emdb-toast-root {
position: fixed; right: 16px; bottom: 16px; z-index: 99999;
diff --git a/em_database/static/settings.js b/em_database/static/settings.js
new file mode 100644
index 0000000..41bacd0
--- /dev/null
+++ b/em_database/static/settings.js
@@ -0,0 +1,134 @@
+// settings.js - an editable settings panel (what display(em_database.settings) shows).
+//
+// Edit the data directory directly: type a path and Save (persist), Use for
+// session (in-memory), or Reset to the default. Also shows the settings-file
+// location and the shared-then-user search order.
+
+function esc(value) {
+ return String(value).replace(/[&<>"']/g, function (c) {
+ return { "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c];
+ });
+}
+
+function el(tag, cls, html) {
+ const n = document.createElement(tag);
+ if (cls) n.className = cls;
+ if (html != null) n.innerHTML = html;
+ return n;
+}
+
+function render({ model, el: root }) {
+ root.classList.add("emdb", "emdb-settings-host");
+
+ let nonce = 0;
+ function cmd(action, extra) {
+ model.set("_command", Object.assign({ action, nonce: nonce++ }, extra || {}));
+ model.save_changes();
+ }
+
+ function draw() {
+ root.innerHTML = "";
+ const wrap = el("div", "emdb-settings");
+ wrap.appendChild(el("div", "emdb-brand", '◆ Settings'));
+
+ // Data directory field
+ const field = el("div", "emdb-field");
+ field.appendChild(el("div", "emdb-field-label", "Data directory"));
+ // Input + Browse on one line (Browse to the right of the path field).
+ const row = el("div", "emdb-field-row");
+ const input = el("input", "emdb-search");
+ input.type = "text";
+ input.value = model.get("data_dir") || "";
+ input.placeholder = model.get("default_dir") || "";
+ input.spellcheck = false;
+ row.appendChild(input);
+ const browse = el("button", "emdb-browse", "Browse…");
+ browse.title = "Pick any file inside the folder you want; its folder is used";
+ row.appendChild(browse);
+ field.appendChild(row);
+
+ // Client-side picker: a plain file (a normal "Open" dialog, no
+ // "upload" prompt). We use the selected file's parent as the directory. The
+ // absolute path is available on desktop/Electron (File.path); a plain
+ // browser hides it, so we fall back to asking the user to type it.
+ const picker = el("input");
+ picker.type = "file";
+ picker.style.display = "none";
+ field.appendChild(picker);
+ const hint = el("div", "emdb-hint");
+ field.appendChild(hint);
+
+ browse.addEventListener("click", function () { picker.click(); });
+ picker.addEventListener("change", function () {
+ const f = picker.files && picker.files[0];
+ if (f) {
+ if (f.path) { // absolute path available (desktop/Electron)
+ const dir = f.path.slice(0, f.path.length - f.name.length).replace(/[\\/]+$/, "");
+ input.value = dir;
+ hint.textContent = "Selected " + dir + " — click Save to keep it.";
+ } else {
+ hint.textContent =
+ "This browser can’t share the folder’s path — type it above instead.";
+ }
+ }
+ picker.value = ""; // let the same file be picked again
+ });
+
+ const actions = el("div", "emdb-field-actions");
+ const save = el("button", "emdb-dl", "Save");
+ save.title = "Set and remember across sessions";
+ save.addEventListener("click", function () { cmd("save", { data_dir: input.value.trim() }); });
+ const session = el("button", "emdb-copy-btn", "Use for session");
+ session.title = "Set for this session only (not written to disk)";
+ session.addEventListener("click", function () { cmd("session", { data_dir: input.value.trim() }); });
+ const reset = el("button", "emdb-delete", "Reset");
+ reset.title = "Back to the default location";
+ reset.addEventListener("click", function () { cmd("reset"); });
+ actions.appendChild(save);
+ actions.appendChild(session);
+ actions.appendChild(reset);
+ field.appendChild(actions);
+ wrap.appendChild(field);
+
+ const status = model.get("status");
+ if (status) wrap.appendChild(el("div", "emdb-status", esc(status)));
+
+ // Info: settings file + search order
+ const info = el("div", "emdb-settings-info");
+ info.appendChild(el("div", "emdb-kv",
+ 'Settings file' +
+ '' + esc(model.get("config_path") || "") + ""));
+
+ const dirs = model.get("search_dirs") || [];
+ if (dirs.length) {
+ info.appendChild(el("div", "emdb-field-label", "Search order"));
+ const list = el("div", "emdb-searchlist");
+ dirs.forEach(function (d, i) {
+ const last = i === dirs.length - 1;
+ const tag = last
+ ? 'downloads here'
+ : 'shared';
+ list.appendChild(el("div", "emdb-searchdir", esc(d) + " " + tag));
+ });
+ info.appendChild(list);
+ }
+ wrap.appendChild(info);
+ root.appendChild(wrap);
+ }
+
+ const redraw = function () { draw(); };
+ model.on("change:data_dir", redraw);
+ model.on("change:status", redraw);
+ model.on("change:search_dirs", redraw);
+ model.on("change:config_path", redraw);
+ draw();
+
+ return function () {
+ model.off("change:data_dir", redraw);
+ model.off("change:status", redraw);
+ model.off("change:search_dirs", redraw);
+ model.off("change:config_path", redraw);
+ };
+}
+
+export default { render };
diff --git a/em_database/tests/conftest.py b/em_database/tests/conftest.py
new file mode 100644
index 0000000..68e55c5
--- /dev/null
+++ b/em_database/tests/conftest.py
@@ -0,0 +1,21 @@
+"""Shared test fixtures.
+
+Every test runs against an isolated, empty settings file (in a tmp dir) and with
+the legacy ``EM_DATABASE_DATA_DIR`` env var cleared, so the developer's real
+settings never affect a test and a test never writes to the real config.
+"""
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _isolate_settings(tmp_path, monkeypatch):
+ from em_database import config
+
+ monkeypatch.setenv("EM_DATABASE_CONFIG", str(tmp_path / "em_database_settings.yaml"))
+ # Point the system config at a nonexistent file and clear the shared/data env
+ # vars, so the developer's real machine config never leaks into a test.
+ monkeypatch.setenv("EM_DATABASE_SYSTEM_CONFIG", str(tmp_path / "system.yaml"))
+ monkeypatch.delenv("EM_DATABASE_DATA_DIR", raising=False)
+ monkeypatch.delenv("EM_DATABASE_SHARED_DIR", raising=False)
+ config.settings.reload() # re-seed from the isolated (empty) config
+ yield
diff --git a/em_database/tests/test_settings.py b/em_database/tests/test_settings.py
new file mode 100644
index 0000000..36b1806
--- /dev/null
+++ b/em_database/tests/test_settings.py
@@ -0,0 +1,66 @@
+"""Tests for the matplotlib-style settings object and the YAML config file.
+
+The autouse fixture in conftest.py isolates each test to an empty settings file
+and clears the legacy env var, so these exercise the mechanism in isolation.
+"""
+import em_database
+from em_database import config
+
+
+def test_defaults_when_nothing_configured():
+ assert em_database.get_data_dir() == config._default_data_dir()
+ assert em_database.settings["data_dir"] == config._default_data_dir()
+
+
+def test_live_object_is_immediate_like_rcparams(tmp_path):
+ em_database.settings["data_dir"] = str(tmp_path / "live")
+ assert em_database.get_data_dir() == str(tmp_path / "live") # no save needed
+ assert not config.config_path().exists() # not persisted
+
+
+def test_set_data_dir_persists_across_sessions(tmp_path):
+ target = str(tmp_path / "data")
+ em_database.set_data_dir(target) # persist=True by default
+ assert em_database.get_data_dir() == target
+ assert config._read_file()["data_dir"] == target
+ config.settings.reload() # a fresh "session"
+ assert em_database.get_data_dir() == target
+
+
+def test_set_data_dir_session_only_is_not_persisted(tmp_path):
+ target = str(tmp_path / "x")
+ em_database.set_data_dir(target, persist=False)
+ assert em_database.get_data_dir() == target
+ assert not config.config_path().exists()
+ config.settings.reload()
+ assert em_database.get_data_dir() == config._default_data_dir() # forgotten
+
+
+def test_reset_clears_the_saved_choice(tmp_path):
+ em_database.set_data_dir(str(tmp_path))
+ em_database.reset_data_dir()
+ assert em_database.get_data_dir() == config._default_data_dir()
+ config.settings.reload()
+ assert em_database.get_data_dir() == config._default_data_dir() # stays reset
+
+
+def test_generic_setting_roundtrips():
+ em_database.set_setting("quality", "high")
+ assert em_database.get_setting("quality") == "high"
+ config.settings.reload()
+ assert em_database.get_setting("quality") == "high" # persisted
+
+
+def test_saving_other_settings_keeps_the_default_dynamic():
+ """Persisting an unrelated setting must NOT freeze the default data_dir into
+ the file - otherwise the default would stop being obeyed."""
+ em_database.set_setting("quality", "high") # triggers a save()
+ stored = config._read_file()
+ assert stored == {"quality": "high"} # data_dir default not written
+ assert em_database.get_data_dir() == config._default_data_dir()
+
+
+def test_legacy_env_var_still_seeds(tmp_path, monkeypatch):
+ monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path / "fromenv"))
+ config.settings.reload() # a fresh "import" with the env var set
+ assert em_database.get_data_dir() == str(tmp_path / "fromenv")
diff --git a/em_database/tests/test_shared_data.py b/em_database/tests/test_shared_data.py
new file mode 100644
index 0000000..7beaf71
--- /dev/null
+++ b/em_database/tests/test_shared_data.py
@@ -0,0 +1,65 @@
+"""Tests for the ~/.em_database settings folder and system-wide data resolution.
+
+The conftest fixture isolates the user + system config and clears the shared/data
+env vars, so these exercise the resolution logic in isolation (no network - the
+"downloads" here find a pre-placed file).
+"""
+import os
+
+import em_database
+from em_database import catalogue, config
+from em_database.tests.test_load_data import TINY_DATASET
+
+
+def _dataset():
+ return catalogue.resolve(TINY_DATASET)
+
+
+def test_settings_live_in_dot_em_database(monkeypatch):
+ monkeypatch.delenv("EM_DATABASE_CONFIG", raising=False) # use the real default
+ path = config.config_path()
+ assert path.parent.name == ".em_database"
+ assert path.name == "settings.yaml"
+
+
+def test_shared_dir_is_searched_before_the_user_dir(tmp_path, monkeypatch):
+ shared, user = tmp_path / "shared", tmp_path / "user"
+ shared.mkdir()
+ user.mkdir()
+ monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared))
+ em_database.set_data_dir(str(user), persist=False)
+
+ ds = _dataset()
+ assert ds.filepath() is None # nowhere yet
+ (user / ds.file).write_bytes(b"user")
+ assert ds.filepath() == str(user / ds.file) # found in the user dir
+ (shared / ds.file).write_bytes(b"shared")
+ assert ds.filepath() == str(shared / ds.file) # shared/system wins
+
+
+def test_download_uses_a_shared_copy_without_refetching(tmp_path, monkeypatch):
+ shared, user = tmp_path / "shared", tmp_path / "user"
+ shared.mkdir()
+ user.mkdir()
+ monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared))
+ em_database.set_data_dir(str(user), persist=False)
+
+ ds = _dataset()
+ (shared / ds.file).write_bytes(b"payload") # pre-installed system-wide
+ path = ds.download(background=False)
+ assert path == str(shared / ds.file) # used the shared copy
+ assert not (user / ds.file).exists() # nothing downloaded to the user dir
+
+
+def test_search_order_is_shared_then_user(tmp_path, monkeypatch):
+ a, b = tmp_path / "a", tmp_path / "b"
+ monkeypatch.setenv("EM_DATABASE_SHARED_DIR", os.pathsep.join([str(a), str(b)]))
+ em_database.set_data_dir(str(tmp_path / "user"), persist=False)
+ assert config.data_search_dirs() == [str(a), str(b), str(tmp_path / "user")]
+
+
+def test_system_config_file_contributes_shared_dirs(tmp_path, monkeypatch):
+ system_file = tmp_path / "system.yaml"
+ system_file.write_text(f"data_dir: {tmp_path / 'sitewide'}\n", encoding="utf-8")
+ monkeypatch.setenv("EM_DATABASE_SYSTEM_CONFIG", str(system_file))
+ assert str(tmp_path / "sitewide") in config.shared_data_dirs()
diff --git a/em_database/tests/test_widget.py b/em_database/tests/test_widget.py
index 3c8ad73..489d2c6 100644
--- a/em_database/tests/test_widget.py
+++ b/em_database/tests/test_widget.py
@@ -42,8 +42,8 @@ def test_catalogue_entry_has_expected_fields():
assert isinstance(row["downloaded"], bool)
-def test_catalogue_downloaded_flag_tracks_the_file(tmp_path, monkeypatch):
- monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path))
+def test_catalogue_downloaded_flag_tracks_the_file(tmp_path):
+ em_database.set_data_dir(str(tmp_path), persist=False)
ds = catalogue.resolve(TINY_DATASET)
assert catalogue.entry(TINY_DATASET, ds)["downloaded"] is False
(tmp_path / ds.file).write_bytes(b"x") # pretend it is downloaded
@@ -121,8 +121,8 @@ def test_search_blob_includes_authors_and_affiliation():
assert "4d-stem" in row["search"] # technique
-def test_delete_removes_downloaded_file(tmp_path, monkeypatch):
- monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path))
+def test_delete_removes_downloaded_file(tmp_path):
+ em_database.set_data_dir(str(tmp_path), persist=False)
ds = catalogue.resolve(TINY_DATASET)
(tmp_path / ds.file).write_bytes(b"x")
assert ds.filepath() is not None
@@ -189,6 +189,43 @@ def _boom(_dataset):
assert "text/plain" in bundle
+def test_settings_widget_edits_and_persists(tmp_path):
+ pytest.importorskip("anywidget")
+ from em_database import config
+ from em_database.widget import settings_widget
+
+ widget = settings_widget()
+ assert widget.data_dir == em_database.get_data_dir()
+
+ widget._command = {"action": "save", "data_dir": str(tmp_path / "d"), "nonce": 1}
+ assert em_database.get_data_dir() == str(tmp_path / "d")
+ assert config._read_file()["data_dir"] == str(tmp_path / "d") # persisted
+ assert widget.data_dir == str(tmp_path / "d")
+
+ widget._command = {"action": "session", "data_dir": str(tmp_path / "s"), "nonce": 2}
+ assert em_database.get_data_dir() == str(tmp_path / "s")
+ assert config._read_file()["data_dir"] == str(tmp_path / "d") # NOT persisted
+
+ widget._command = {"action": "reset", "nonce": 3}
+ assert em_database.get_data_dir() == config._default_data_dir()
+
+
+def test_settings_display_is_a_widget():
+ pytest.importorskip("anywidget")
+ bundle = em_database.settings._repr_mimebundle_()
+ mimes = bundle[0] if isinstance(bundle, tuple) else bundle
+ assert "application/vnd.jupyter.widget-view+json" in mimes
+
+
+def test_notebook_detection_and_colab_enable_are_safe():
+ """The frontend helpers must be no-ops off a notebook (e.g. under pytest),
+ so nothing breaks when em_database is imported in plain Python."""
+ import em_database.widget as widget_mod
+ assert widget_mod._in_notebook() is False
+ widget_mod._enable_colab_widgets() # must not raise when not on Colab
+ widget_mod._prepare_frontend() # idempotent, safe
+
+
def test_attach_toast_is_noop_outside_jupyter():
"""Outside a Jupyter kernel there is no toast, so a bare download is
unaffected."""
@@ -224,8 +261,8 @@ def test_download_toasts_plumbing():
@pytest.mark.slow
-def test_widget_download_end_to_end(tmp_path, monkeypatch):
- monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path))
+def test_widget_download_end_to_end(tmp_path):
+ em_database.set_data_dir(str(tmp_path), persist=False)
widget = _browser()
future = widget._start_download(TINY_DATASET)
assert future is not None
diff --git a/em_database/widget.py b/em_database/widget.py
index 3c4eec6..6672cb3 100644
--- a/em_database/widget.py
+++ b/em_database/widget.py
@@ -48,6 +48,32 @@ def _quiet_pooch():
pass
+_colab_enabled = False
+
+
+def _enable_colab_widgets():
+ """On Google Colab, third-party (anywidget) widgets render only once the
+ custom widget manager is enabled - do it once, automatically. No-op anywhere
+ else, so widgets work out of the box on Colab and Jupyter alike.
+ """
+ global _colab_enabled
+ if _colab_enabled:
+ return
+ try:
+ from google.colab import output # importable only on Colab
+ output.enable_custom_widget_manager()
+ except Exception:
+ pass
+ finally:
+ _colab_enabled = True
+
+
+def _prepare_frontend():
+ """Everything that should happen before a widget renders."""
+ _quiet_pooch()
+ _enable_colab_widgets()
+
+
class DownloadCancelled(Exception):
"""Raised inside pooch's stream when the user cancels a download.
@@ -334,7 +360,7 @@ def card(dataset):
This backs ``display(dataset)`` / a dataset being the last line in a cell.
Requires anywidget (``pip install em-database[widget]``).
"""
- _quiet_pooch()
+ _prepare_frontend()
global _card_class
if _card_class is None:
try:
@@ -354,7 +380,7 @@ def browse(**kwargs):
every dataset grouped by technique, shows which are downloaded, reveals full
metadata on hover, and downloads on click with a live progress toast.
"""
- _quiet_pooch()
+ _prepare_frontend()
global _browser_class
if _browser_class is None:
try:
@@ -371,6 +397,78 @@ def browse(**kwargs):
# Global toasts: a bare ``ds.download()`` in Jupyter pops a cancelable toast
# ---------------------------------------------------------------------------
+def _make_settings_class():
+ """Build the ``SettingsWidget`` class, importing anywidget lazily."""
+ import anywidget
+ import traitlets
+
+ from em_database import config
+
+ class SettingsWidget(anywidget.AnyWidget):
+ """An editable panel for em_database's settings (the data directory)."""
+
+ _esm = _STATIC / "settings.js"
+ _css = _STATIC / "browser.css"
+
+ data_dir = traitlets.Unicode().tag(sync=True)
+ default_dir = traitlets.Unicode().tag(sync=True)
+ config_path = traitlets.Unicode().tag(sync=True)
+ search_dirs = traitlets.List().tag(sync=True)
+ status = traitlets.Unicode().tag(sync=True)
+ _command = traitlets.Dict().tag(sync=True)
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self._refresh()
+ self.observe(self._on_command, names="_command")
+
+ def _refresh(self, status=""):
+ self.data_dir = config.data_dir()
+ self.default_dir = config._default_data_dir()
+ self.config_path = str(config.config_path())
+ self.search_dirs = list(config.data_search_dirs())
+ self.status = status
+
+ def _on_command(self, change):
+ command = change.get("new") or {}
+ action = command.get("action")
+ value = str(command.get("data_dir", "")).strip()
+ if action == "save" and value:
+ config.settings["data_dir"] = value
+ config.settings.save()
+ self._refresh(status=f"Saved to {config.config_path()}")
+ elif action == "session" and value:
+ config.settings["data_dir"] = value
+ self._refresh(status="Set for this session (not saved)")
+ elif action == "reset":
+ config.settings.reset("data_dir")
+ self._refresh(status="Reset to the default location")
+
+ return SettingsWidget
+
+
+_settings_class = None
+
+
+def settings_widget():
+ """Return an interactive widget for editing em_database's settings (Jupyter).
+
+ Backs ``display(em_database.settings)`` and ``em_database.settings.widget()``.
+ Requires anywidget (``pip install em-database[widget]``).
+ """
+ _prepare_frontend()
+ global _settings_class
+ if _settings_class is None:
+ try:
+ _settings_class = _make_settings_class()
+ except ImportError as error:
+ raise ImportError(
+ "The settings widget needs anywidget. Install it with "
+ "`pip install em-database[widget]` (or `pip install anywidget`)."
+ ) from error
+ return _settings_class()
+
+
def _make_toasts_class():
"""Build the singleton ``DownloadToasts`` widget, importing anywidget lazily."""
import anywidget
@@ -450,19 +548,24 @@ def _set_error(self, token, label, message):
_toasts_class = None
-def _in_jupyter():
- """True only in a Jupyter kernel (where widgets render), not plain Python."""
+def _in_notebook():
+ """True in a notebook frontend that can render widgets (Jupyter, Colab,
+ VS Code, ...), False in plain Python or a terminal IPython."""
try:
from IPython import get_ipython
ip = get_ipython()
- return ip is not None and ip.__class__.__name__ == "ZMQInteractiveShell"
+ if ip is None:
+ return False
+ # ZMQInteractiveShell = Jupyter; "Shell" = Colab; exclude only the
+ # terminal shell, which cannot render widgets.
+ return ip.__class__.__name__ != "TerminalInteractiveShell"
except Exception:
return False
def _get_toasts():
"""Return the singleton toasts widget, or None if a toast can't be shown
- (not in Jupyter, or anywidget missing).
+ (not in a notebook, or anywidget missing).
The widget is re-displayed on every call so it re-anchors in the current
cell: a widget view is tied to a cell's output, so clearing or re-running
@@ -471,9 +574,9 @@ def _get_toasts():
never duplicates the toasts.
"""
global _toasts, _toasts_class
- if not _in_jupyter():
+ if not _in_notebook():
return None
- _quiet_pooch()
+ _prepare_frontend()
try:
if _toasts_class is None:
_toasts_class = _make_toasts_class()
diff --git a/examples/changing_download_location.py b/examples/changing_download_location.py
index c847ba7..17b539f 100644
--- a/examples/changing_download_location.py
+++ b/examples/changing_download_location.py
@@ -1,26 +1,50 @@
"""
-Changing the Download Location
-==============================
+Settings and the Download Location
+==================================
-This example demonstrates how to change the default download location for datasets
-in the `em_database` package. By default, datasets are downloaded to a predefined
-directory, but users can customize this location as needed.
+Configuration for ``em_database`` lives in a live, matplotlib-``rcParams``-style
+object, ``em_database.settings``, seeded at import from a YAML file in
+``~/.em_database/settings.yaml``. This example shows how to read it, change the
+data directory for the session or for good, and reset it.
"""
-from em_database import get_data_dir, set_data_dir, reset_data_dir
-
-
-
-# Display the current download directory
-print("Current download directory:", get_data_dir())
-# Set a new download directory
-new_directory = "/path/to/your/custom/directory" # Replace with your desired path
-set_data_dir(new_directory)
-print("Updated download directory:", get_data_dir())
-# Reset to the default download directory
-reset_data_dir()
-print("Reset download directory:", get_data_dir())
-
-
-
-
+import em_database
+
+# %%
+# The data directory is where datasets download to. It defaults to
+# ``~/em_database``.
+print("Current download directory:", em_database.get_data_dir())
+
+# %%
+# Change it for this session only (in memory, not written to disk).
+em_database.set_data_dir("/path/to/scratch", persist=False)
+print("Session directory:", em_database.get_data_dir())
+
+# %%
+# Change it and remember the choice across sessions (writes
+# ``~/.em_database/settings.yaml``). ``set_data_dir`` persists by default; the
+# equivalent low-level form is ``em_database.settings[...] = ...; save()``.
+em_database.set_data_dir("/big/disk/em_data") # set + persist
+# em_database.settings["data_dir"] = "/big/disk/em_data" # the same thing
+# em_database.settings.save()
+print("Persisted directory:", em_database.get_data_dir())
+
+# %%
+# Reset to the default location, forgetting the saved choice.
+em_database.reset_data_dir()
+print("Reset directory:", em_database.get_data_dir())
+
+# %%
+# Any setting can be stored, not just the data directory.
+em_database.set_setting("quality", "high")
+print("quality:", em_database.get_setting("quality"))
+
+# %%
+# Datasets can also be installed system-wide and shared by every user. Point
+# ``EM_DATABASE_SHARED_DIR`` (or a ``shared_data_dirs`` setting, or a system
+# config file) at the shared location, and ``download()`` / ``filepath()`` will
+# find a pre-installed copy there before downloading into your own directory.
+
+# %%
+# Clean up so running this example leaves no settings behind.
+em_database.settings.reset()