Settings in ~/.em_database + system-wide data resolution

README.md

@@ -35,17 +35,49 @@ Call `path.done()` to check progress without blocking, or `path.result()` to wai
 explicitly.  Pass `download(background=False)` to block and get the path as a plain
 string instead.
 
-The download location defaults to `~/em_database` and can be changed either for the
-session or through the `EM_DATABASE_DATA_DIR` environment variable:
+## Settings
+
+Configuration lives in a live, matplotlib-`rcParams`-style object,
+`em_database.settings`, seeded at import from `~/.em_database/settings.yaml`.
+Change it in memory for an immediate effect, and persist it to remember the
+choice across sessions:
 
 ```python
 import em_database
 
-em_database.set_data_dir("/path/to/somewhere")
-em_database.get_data_dir()
-em_database.reset_data_dir()
+em_database.settings["data_dir"] = "/big/disk/em_data"  # takes effect now
+em_database.settings.save()                             # remember it next time
 ```
 
+In Jupyter you can also edit them interactively — `display(em_database.settings)`
+renders a panel to set (and save) the data directory directly.
+
+The data directory defaults to `~/em_database`. Convenience helpers wrap the
+common case — `set_data_dir` persists by default:
+
+```python
+em_database.get_data_dir()                       # current location
+em_database.set_data_dir("/big/disk/em_data")    # set + persist
+em_database.set_data_dir("/scratch", persist=False)  # one-off, in-memory only
+em_database.reset_data_dir()                     # back to the default, forget the choice
+```
+
+No environment variable is needed. Only values you explicitly set are written to
+the file, so the default is never frozen in — it keeps being obeyed even if it
+changes. For a one-off override (e.g. CI) the legacy `EM_DATABASE_DATA_DIR` is
+still honored, and `EM_DATABASE_CONFIG` relocates the settings file.
+
+### Shared, system-wide data
+
+Datasets can be installed once for every user. `download()` and `filepath()`
+look in the shared/system locations **first**, then your own data directory, and
+only download (into your directory) if the file is nowhere to be found — so a
+shared copy is reused instead of refetched. Shared locations come from the
+`EM_DATABASE_SHARED_DIR` environment variable (an `os.pathsep`-separated list), a
+`shared_data_dirs` list in your settings, or a system config file
+(`/etc/em_database/settings.yaml`, or `%PROGRAMDATA%\em_database\settings.yaml`
+on Windows).
+
 ## Adding a dataset
 
 Datasets are described by a YAML file in `em_database/datasets/`, one entry per file,

em_database/__init__.py

@@ -1,19 +1,12 @@
 ### Example datasets ###
 import os
+from em_database.config import settings
 from em_database.downloadable_dataset import DownloadableDataset
 from em_database._create_stubs import build_docstring
 from em_database import data
 __all__ = []
 
 
-if "EM_DATABASE_DATA_DIR" not in os.environ:
-    # set the default dir to User's home directory + "/emdata"
-    os.environ["EM_DATABASE_DATA_DIR"] = os.path.join(
-        os.path.expanduser("~"),"em_database"
-    )
-
-
-
 def get_data_dir():
     """
     Get the directory where example datasets are stored.
@@ -23,26 +16,42 @@ def get_data_dir():
     str
         Path to the example datasets directory.
     """
-    return  os.environ["EM_DATABASE_DATA_DIR"]
+    from em_database import config
+    return config.data_dir()
 
-def set_data_dir(path: str):
+def set_data_dir(path: str, persist: bool = True):
     """
     Set the directory where example datasets are stored.
 
     Parameters
     ----------
     path : str
         Path to the desired example datasets directory.
+    persist : bool, optional
+        If True (the default), remember the choice across sessions by writing it
+        to the settings file. Pass False for a one-off, in-memory change.
     """
-    os.environ["EM_DATABASE_DATA_DIR"] = path
+    settings["data_dir"] = str(path)
+    if persist:
+        settings.save()
 
 def reset_data_dir():
     """
-    Reset the example datasets directory to the default location.
+    Reset the example datasets directory to the default location, clearing any
+    saved choice.
     """
-    os.environ["EM_DATABASE_DATA_DIR"] = os.path.join(
-        os.path.expanduser("~"),"em_database"
-    )
+    settings.reset("data_dir")
+
+
+def get_setting(key: str, default=None):
+    """Read a value from :data:`em_database.settings`."""
+    return settings.get(key, default)
+
+def set_setting(key: str, value, persist: bool = True):
+    """Set a value in :data:`em_database.settings`, persisting it by default."""
+    settings[key] = value
+    if persist:
+        settings.save()
 
 
 def browse(**kwargs):
@@ -60,7 +69,8 @@ def browse(**kwargs):
     return _browse(**kwargs)
 
 
-__all__ =  ['get_data_dir', 'set_data_dir', 'reset_data_dir', 'browse', "data"]
+__all__ =  ['get_data_dir', 'set_data_dir', 'reset_data_dir',
+            'get_setting', 'set_setting', 'settings', 'browse', "data"]
 
 
 # Let ``display(em_database)`` render the browser. Reassigning the module's

em_database/config.py

@@ -0,0 +1,213 @@
+"""Runtime settings for em_database, in the style of matplotlib's ``rcParams``.
+
+``em_database.settings`` is a live dict, seeded at import from a YAML file (or
+the built-in defaults). Change it in memory::
+
+    import em_database
+    em_database.settings["data_dir"] = "/big/disk/em_data"
+
+and it takes effect immediately. To remember the choice across sessions, persist
+it::
+
+    em_database.settings.save()
+
+Settings live in a ``~/.em_database`` folder (``~/.em_database/settings.yaml``),
+or wherever ``EM_DATABASE_CONFIG`` points. No environment variable is *needed* to
+configure em_database - the settings object is the source of truth; the old
+``EM_DATABASE_DATA_DIR`` is still honored as a seed for backward compatibility.
+
+Data can also be installed **system-wide** and shared by every user. When
+resolving where a dataset lives, em_database looks in the shared/system
+locations first, then the user's own data directory, and only downloads (into
+the user directory) if it is nowhere to be found. Shared locations come from a
+system config file (see :func:`system_config_path`), the ``EM_DATABASE_SHARED_DIR``
+environment variable (an ``os.pathsep``-separated list), or a ``shared_data_dirs``
+list in the user's settings.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import yaml
+
+
+def _default_data_dir() -> str:
+    return os.path.join(os.path.expanduser("~"), "em_database")
+
+
+#: Built-in defaults - the fallback when nothing is configured.
+DEFAULTS: dict = {
+    "data_dir": _default_data_dir(),
+}
+
+
+def config_dir() -> Path:
+    """The ``~/.em_database`` folder that holds the settings file."""
+    return Path(os.path.expanduser("~")) / ".em_database"
+
+
+def config_path() -> Path:
+    """Path to the YAML settings file.
+
+    ``EM_DATABASE_CONFIG`` overrides the location; otherwise it is
+    ``~/.em_database/settings.yaml``.
+    """
+    override = os.environ.get("EM_DATABASE_CONFIG")
+    if override:
+        return Path(override)
+    return config_dir() / "settings.yaml"
+
+
+def system_config_path() -> Path | None:
+    """Path to the machine-wide settings file (for system-wide installs).
+
+    ``EM_DATABASE_SYSTEM_CONFIG`` overrides it; otherwise it is
+    ``/etc/em_database/settings.yaml`` (POSIX) or
+    ``%PROGRAMDATA%\\em_database\\settings.yaml`` (Windows).
+    """
+    override = os.environ.get("EM_DATABASE_SYSTEM_CONFIG")
+    if override:
+        return Path(override)
+    if os.name == "nt":
+        base = os.environ.get("PROGRAMDATA")
+        return Path(base) / "em_database" / "settings.yaml" if base else None
+    return Path("/etc/em_database/settings.yaml")
+
+
+def _read_yaml(path: Path | None) -> dict:
+    """Parse a YAML file into a dict, or ``{}`` if missing/unreadable/malformed."""
+    if path is None:
+        return {}
+    try:
+        with open(path, encoding="utf-8") as handle:
+            data = yaml.safe_load(handle)
+        return data if isinstance(data, dict) else {}
+    except FileNotFoundError:
+        return {}
+    except Exception:
+        # A malformed settings file must not break importing the package.
+        return {}
+
+
+def _read_file() -> dict:
+    """The user settings stored on disk, or ``{}``."""
+    return _read_yaml(config_path())
+
+
+def _read_system() -> dict:
+    """The machine-wide settings, or ``{}``."""
+    return _read_yaml(system_config_path())
+
+
+def _write_file(data: dict) -> None:
+    path = config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as handle:
+        yaml.safe_dump(dict(data), handle, default_flow_style=False, sort_keys=True)
+
+
+def _seed(settings: "Settings") -> None:
+    """(Re)seed a settings object: defaults < file < legacy env var."""
+    settings.clear()
+    settings.update(DEFAULTS)
+    settings.update(_read_file())  # the file overrides the defaults
+    # Backward compatibility: seed from the old env var if it is set.
+    env_dir = os.environ.get("EM_DATABASE_DATA_DIR")
+    if env_dir:
+        settings["data_dir"] = env_dir
+
+
+class Settings(dict):
+    """A live settings mapping that can persist itself to the YAML file."""
+
+    def save(self) -> None:
+        """Persist the settings to the YAML file.
+
+        Only values that DIFFER from the built-in defaults are written, so a
+        default (like the data directory) is never frozen into the file: it stays
+        dynamic and keeps being obeyed across sessions, machines, or a change to
+        the default itself. When nothing differs, any stale file is removed.
+        """
+        delta = {key: value for key, value in self.items() if DEFAULTS.get(key) != value}
+        path = config_path()
+        if delta:
+            _write_file(delta)
+        elif path.exists():
+            path.unlink()
+
+    def reload(self) -> None:
+        """Re-seed from the built-in defaults and the on-disk file, discarding
+        any in-memory changes."""
+        _seed(self)
+
+    def reset(self, key: str | None = None) -> None:
+        """Reset a key (or everything) to its default, in memory and on disk."""
+        if key is None:
+            self.clear()
+            self.update(DEFAULTS)
+        elif key in DEFAULTS:
+            self[key] = DEFAULTS[key]
+        else:
+            self.pop(key, None)
+        self.save()  # delta-save drops the now-default key from the file
+
+    def widget(self):
+        """Return an interactive widget for editing the settings (Jupyter)."""
+        from em_database.widget import settings_widget
+        return settings_widget()
+
+    def _repr_mimebundle_(self, **kwargs):
+        """Render as an editable settings widget in Jupyter (falls back to the
+        plain dict repr if anywidget is not installed)."""
+        try:
+            from em_database.widget import settings_widget
+            widget = settings_widget()
+        except Exception:
+            return {"text/plain": repr(dict(self))}
+        return widget._repr_mimebundle_(**kwargs)
+
+
+#: The live settings object (seeded once at import).
+settings = Settings()
+_seed(settings)
+
+
+def data_dir() -> str:
+    """The user's data directory - where downloads are written."""
+    return str(settings.get("data_dir") or _default_data_dir())
+
+
+def shared_data_dirs() -> list[str]:
+    """System-wide / shared data locations, checked before the user's dir.
+
+    Order: ``EM_DATABASE_SHARED_DIR`` (an ``os.pathsep`` list), then the user's
+    ``shared_data_dirs`` setting, then the system config file's ``data_dir`` and
+    ``shared_data_dirs``. Duplicates are removed, order preserved.
+    """
+    dirs: list[str] = []
+    env = os.environ.get("EM_DATABASE_SHARED_DIR")
+    if env:
+        dirs += [d for d in env.split(os.pathsep) if d]
+    dirs += [str(d) for d in (settings.get("shared_data_dirs") or [])]
+    system = _read_system()
+    if system.get("data_dir"):
+        dirs.append(str(system["data_dir"]))
+    dirs += [str(d) for d in (system.get("shared_data_dirs") or [])]
+    seen: set[str] = set()
+    unique: list[str] = []
+    for d in dirs:
+        if d not in seen:
+            seen.add(d)
+            unique.append(d)
+    return unique
+
+
+def data_search_dirs() -> list[str]:
+    """Everywhere to look for an existing dataset: shared/system dirs first,
+    then the user's data directory."""
+    dirs = shared_data_dirs()
+    user = data_dir()
+    if user not in dirs:
+        dirs.append(user)
+    return dirs