diff --git a/.github/ISSUE_TEMPLATE/new_dataset.yaml b/.github/ISSUE_TEMPLATE/new_dataset.yaml index 2c4bfe8..333f5d8 100644 --- a/.github/ISSUE_TEMPLATE/new_dataset.yaml +++ b/.github/ISSUE_TEMPLATE/new_dataset.yaml @@ -27,11 +27,19 @@ body: placeholder: ex. Jane Doe validations: required: true + - type: input + id: affiliation + attributes: + label: --Affiliation-- + description: Which institution or company are you at? + placeholder: ex. University of Wisconsin-Madison + validations: + required: true - type: input id: url attributes: label: --URL-- - description: Please provide a URL where the dataset can be downloaded. This should be a direct link to the file itself. + description: Please provide a URL where the dataset can be downloaded. This must be a direct link to the file itself - the file name is taken from the end of it. placeholder: ex. https://zenodo.org/records/15490547/files/smallPtychography.hspy validations: required: true @@ -106,11 +114,12 @@ body: description: Select the technique used to acquire the dataset. options: - 4D-STEM - - EBSD - - IN SITU TEM - - SEM - EELS - - XEDS + - EDS + - EBSD + - STEM + - In-situ TEM + - Cryo-EM - Other - type: input id: tags diff --git a/.github/scripts/issue_to_yaml.py b/.github/scripts/issue_to_yaml.py index 7ce283c..f75e160 100644 --- a/.github/scripts/issue_to_yaml.py +++ b/.github/scripts/issue_to_yaml.py @@ -1,66 +1,130 @@ -import sys +"""Turn a filled-in "New Dataset" issue into a dataset YAML file. + +Run by ``.github/workflows/on_new_issue.yml``. The output is validated against +``em_database/datasets/json-schema.json`` before it is written, so a malformed +issue fails here rather than in the pull request the workflow opens. +""" + import re -import yaml +import sys +import urllib.request from pathlib import Path +import jsonschema +import yaml + +from em_database.metadata import check_vendor, load_schema, load_vendors + +FIELDS = { + "Dataset Name": r"--Dataset Name--\s*(.*)", + "Author": r"--Author--\s*(.*)", + "Affiliation": r"--Affiliation--\s*(.*)", + "URL": r"--URL--\s*(.*)", + "Checksum": r"--Checksum--\s*(.*)", + "Description": r"--Description--\s*([\s\S]*?)--Detector Manufacturer--", + "Detector Manufacturer": r"--Detector Manufacturer--\s*(.*)", + "Detector Model": r"Detector Model\s*(.*)", + "Microscope Vendor": r"Microscope Vendor\s*(.*)", + "Microscope Model": r"Microscope Model\s*(.*)", + "Accelerating Voltage": r"Accelerating Voltage\s*(.*)", + "Dataset License": r"Dataset License\s*(.*)", + "Technique": r"Technique\s*(.*)", + "Tags": r"Tags\s*(.*)", +} + + +def clean(value): + """Trim what GitHub adds around an issue-form answer. + + Answers are separated by ``###`` headings, and the multi-line ones run up + to the next heading; an optional field left blank comes through as the + literal ``_No response_``. + """ + value = value.strip().rstrip("#").strip() + return "" if value == "_No response_" else value + + def parse_issue_body(text): - # Simple regex-based parser for the fields - fields = { - "Dataset Name": r"--Dataset Name--\s*(.*)", - "Author": r"--Author--\s*(.*)", - "URL": r"--URL--\s*(.*)", - "Checksum": r"--Checksum--\s*(.*)", - "Description": r"--Description--\s*([\s\S]*?)--Detector Manufacturer--", - "Detector Manufacturer": r"--Detector Manufacturer--\s*(.*)", - "Detector Model": r"Detector Model\s*(.*)", - "Microscope Vendor": r"Microscope Vendor\s*(.*)", - "Microscope Model": r"Microscope Model\s*(.*)", - "Accelerating Voltage": r"Accelerating Voltage\s*(.*)", - "Dataset License": r"Dataset License\s*(.*)", - "Technique": r"Technique\s*(.*)", - "Tags": r"Tags\s*(.*)" - } data = {} - for key, pattern in fields.items(): - m = re.search(pattern, text) - if m: - data[key] = m.group(1).strip() - else: - data[key] = "" + for key, pattern in FIELDS.items(): + match = re.search(pattern, text) + data[key] = clean(match.group(1)) if match else "" return data + +def content_length(url): + """The file's size in bytes, or None if the server will not say.""" + request = urllib.request.Request( + url, + method="HEAD", + headers={"User-Agent": "em_database (https://github.com/CSSFrancis/em_data)"}, + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + length = response.headers.get("Content-Length") + return int(length) if length else None + except Exception: + return None + + def build_yaml(data): - # Convert tags to list - tags = [t.strip() for t in data["Tags"].split(",") if t.strip()] - # Use author as dataset name (sanitize) - d_name = re.sub(r'\W+', '', data["Dataset Name"]) - yaml_data = { - d_name: { - "description": data["Description"], - "source": data["URL"], - "detector_manufacturer": data["Detector Manufacturer"], - "detector": data["Detector Model"], - "microscope_vendor": data["Microscope Vendor"], - "microscope": data["Microscope Model"], - "voltage": data["Accelerating Voltage"], - "license": data["Dataset License"], - "technique": data["Technique"], - "tags": tags, - "authors": { - data["Author"]: {} - } - } + """``(document, dataset_name)`` for a parsed issue.""" + name = re.sub(r"\W+", "", data["Dataset Name"]) + if not name: + sys.exit("the issue has no dataset name") + # The form asks for a direct link to the file; the YAML wants the directory + # and the file name separately. + url = data["URL"].rstrip("/") + source, _, filename = url.rpartition("/") + if not source or not filename: + sys.exit(f"{data['URL']!r} is not a direct link to a file") + + entry = { + "description": data["Description"], + "source": source, + "checksum": data["Checksum"], + "file": filename, + "size_bytes": content_length(url), + "detector_manufacturer": data["Detector Manufacturer"], + "detector": data["Detector Model"], + "microscope_vendor": data["Microscope Vendor"], + "microscope_model": data["Microscope Model"], + "voltage": data["Accelerating Voltage"], + "license": data["Dataset License"], + "technique": data["Technique"], + "tags": [t.strip() for t in data["Tags"].split(",") if t.strip()], } - return yaml_data, dataset_name + if data["Author"]: + entry["authors"] = {data["Author"]: {"affiliation": data["Affiliation"] or "Unspecified"}} + return {name: {k: v for k, v in entry.items() if v not in (None, "", [])}}, name + + +def report_vendors(entry): + """Print vendor warnings; exit on a name that is a likely misspelling.""" + vendors = load_vendors() + failed = False + for field, known in ( + ("detector_manufacturer", vendors["detector_manufacturer"]), + ("microscope_vendor", vendors["microscope_vendor"]), + ): + result = check_vendor(entry.get(field, ""), known) + if result is None: + continue + level, message = result + print(f"{level}: {field}: {message}") + failed = failed or level == "error" + if failed: + sys.exit("fix the vendor spelling in the issue and reopen it") + if __name__ == "__main__": - issue_file = sys.argv[1] - out_dir = Path(sys.argv[2]) - with open(issue_file) as f: - text = f.read() - data = parse_issue_body(text) - yaml_data, dataset_name = build_yaml(data) + issue_file, out_dir = sys.argv[1], Path(sys.argv[2]) + document, dataset_name = build_yaml(parse_issue_body(Path(issue_file).read_text())) + jsonschema.validate(document, load_schema()) + report_vendors(document[dataset_name]) + out_path = out_dir / f"{dataset_name}.yaml" with open(out_path, "w") as f: f.write("# $schema: ../json-schema.json\n") - yaml.dump(yaml_data, f, sort_keys=False) \ No newline at end of file + yaml.dump(document, f, sort_keys=False) + print(f"wrote {out_path}") diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35be344..2a50ac3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,33 @@ env: MPLBACKEND: agg jobs: + lint: + name: lint and type check + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies and package + run: pip install -U -e .'[dev]' + + - name: ruff check + run: ruff check --output-format=github . + + - name: ruff format + run: ruff format --check --diff . + + - name: basedpyright + run: basedpyright + + - name: dataset stub is current + run: python -m em_database._create_stubs --check + build-with-pip: name: ${{ matrix.os }}-py${{ matrix.python-version }}${{ matrix.LABEL }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/check_sources.yml b/.github/workflows/check_sources.yml new file mode 100644 index 0000000..f08ef21 --- /dev/null +++ b/.github/workflows/check_sources.yml @@ -0,0 +1,29 @@ +name: check sources + +# Every dataset's source URL, checked against the server. This is what rots over +# time - a Zenodo record superseded, a GitHub ref rewritten - and it is invisible +# until someone tries to download. It is one HEAD request per dataset, so it runs +# weekly rather than on the six-way matrix of every push. +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +jobs: + sources: + name: source URLs still resolve + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies and package + run: pip install -U -e .'[dev]' + + - name: Check every source URL + run: pytest -m network -v diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index 3e72189..48cf1eb 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -26,6 +26,13 @@ jobs: run: | make html -C doc + - name: Upload the built site + uses: actions/upload-artifact@v4 + with: + name: html-${{ github.ref_name }} + path: doc/build/html + retention-days: 14 + - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/on_new_issue.yml b/.github/workflows/on_new_issue.yml index 0e7fd03..ca94453 100644 --- a/.github/workflows/on_new_issue.yml +++ b/.github/workflows/on_new_issue.yml @@ -17,6 +17,9 @@ jobs: with: python-version: '3.x' + - name: Install dependencies and package + run: pip install -U -e .'[dev]' + - name: Extract issue body run: echo "${{ github.event.issue.body }}" > issue_body.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2701d84 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.5 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format diff --git a/README.md b/README.md index 9fe94d1..c9bec0d 100644 --- a/README.md +++ b/README.md @@ -29,11 +29,37 @@ s = hs.load(path, lazy=True) ``` By default the download runs on a background thread so a notebook cell returns -immediately. The handle it returns *is* the file path, so you can hand it straight -to a loader as above — it only blocks at the moment the file is actually opened. -Call `path.done()` to check progress without blocking, or `path.result()` to wait -explicitly. Pass `download(background=False)` to block and get the path as a plain -string instead. +immediately. The handle it returns *is* the file path — a `pathlib.Path` subclass +pointing at the file's final location — so you can hand it straight to a loader as +above; it only blocks at the moment the file is actually opened. Read `path.done` +to check progress without blocking, or call `path.result()` to wait explicitly. +`download(background=False)` blocks instead, and returns the same type. + +Any path pointing at the same file waits, however it was built, so a derived path +(`handle.parent / handle.name`, `handle.with_suffix(...)`) behaves too. The +exceptions are `str(handle)` and `Path(handle)`: both hand back an ordinary value +with no download attached, so `hs.load(str(handle))` will *not* wait. Keeping +`str()` non-blocking is deliberate — `repr()` needs it — so pass the handle itself. + +## Finding a dataset + +`search()` is the browser widget's search box, callable from Python; `filter()` +matches named fields. Both return dataset objects, so a result can be downloaded +directly. + +```python +import em_database + +em_database.list_datasets() # everything +em_database.search("amorphous") # any field +em_database.search("jeol eels") # all terms, any field +em_database.filter(technique="4D-STEM", tags="Strain") # exact, case-insensitive +em_database.filter(microscope_vendor=["JEOL", "Hitachi"]) # a list means any of +em_database.filter(downloaded=True) # what is already here +``` + +An unknown field raises rather than being ignored, so a typo cannot quietly return +the whole index. ## Settings @@ -88,11 +114,16 @@ from the top-level key: MyDataset: description: What the data is, how it was acquired and how it is calibrated. source: https://zenodo.org/records//files - checksum: md5: file: MyDataset.zspy - data_size: 1.2 GB + checksum: md5: + size_bytes: 1200000000 technique: 4D-STEM license: CC-BY-4.0 ``` +`size_bytes` is the file's `Content-Length` in bytes; the test suite checks it against +the server on every run. `em_database/datasets/vendors.yaml` lists the microscope +vendors and detector manufacturers already in use - a new one is fine, but a name close +to one already on the list fails CI as a misspelling. + Open an issue with the new dataset template, or add the YAML file directly. diff --git a/doc/source/conf.py b/doc/source/conf.py index 59d83e6..2a7c059 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -5,14 +5,15 @@ import sys from pathlib import Path + # Import and run the build script from em_database._build_docs import ( - parse_datasets, - generate_html_table, + generate_add_dataset_html, + generate_all_data_html, generate_browser_html, + generate_html_table, generate_landing_html, - generate_all_data_html, - generate_add_dataset_html, + parse_datasets, ) # Add project root to path @@ -21,10 +22,10 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'em_database' -copyright = '2026, Carter Francis' -author = 'Carter Francis' -release = '0.4.0' +project = "em_database" +copyright = "2026, Carter Francis" +author = "Carter Francis" +release = "0.4.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration @@ -35,15 +36,14 @@ "sphinx.ext.intersphinx", "sphinx.ext.napoleon", "sphinx_gallery.gen_gallery", - 'sphinx_design', + "sphinx_design", ] -templates_path = ['_templates'] +templates_path = ["_templates"] # intro.rst / datasets.rst are superseded by the generated landing + All Data # app pages; keep the files but leave them out of the build so they don't warn # about being orphaned. -exclude_patterns = ['intro.rst', 'datasets.rst'] - +exclude_patterns = ["intro.rst", "datasets.rst"] # -- Options for HTML output ------------------------------------------------- @@ -87,17 +87,18 @@ # No left sidebar anywhere - keep every page a single, full-width column. html_sidebars = {"**": []} _unused_sidebars = { - "index": [], - "all_data": [], - "add_dataset": [], - "datasets": [], + "index": [], + "all_data": [], + "add_dataset": [], + "datasets": [], } + def build_datasets_html(app, exception): """Generate datasets.html during Sphinx build""" if exception is not None: print(f"Build exception: {exception}") - datasets_path = Path(__file__).parent.parent.parent / 'em_database' / 'datasets' + datasets_path = Path(__file__).parent.parent.parent / "em_database" / "datasets" print(f"Looking for datasets at: {datasets_path.absolute()}") print(f"Path exists: {datasets_path.exists()}") if datasets_path.exists(): @@ -107,9 +108,9 @@ def build_datasets_html(app, exception): print(datasets) html_output = generate_html_table(datasets) - output_path = Path(app.outdir) / 'datasets_db.html' + output_path = Path(app.outdir) / "datasets_db.html" output_path.parent.mkdir(parents=True, exist_ok=True) - with output_path.open('w', encoding='utf-8') as f: + with output_path.open("w", encoding="utf-8") as f: f.write(html_output) # Generated, self-contained Catppuccin "app" pages. Each is written into the @@ -118,20 +119,22 @@ def build_datasets_html(app, exception): # em_database.browse(). Each is guarded so a failure never kills the build. outdir = Path(app.outdir) pages = { - 'index.html': generate_landing_html, - 'all_data.html': generate_all_data_html, - 'add_dataset.html': generate_add_dataset_html, - 'datasets_browser.html': generate_browser_html, + "index.html": generate_landing_html, + "all_data.html": generate_all_data_html, + "add_dataset.html": generate_add_dataset_html, + "datasets_browser.html": generate_browser_html, } for filename, generator in pages.items(): try: - (outdir / filename).write_text(generator(), encoding='utf-8') + (outdir / filename).write_text(generator(), encoding="utf-8") print(f"Wrote {filename}") except Exception as e: # pragma: no cover - keep the build alive print(f"Could not build {filename}: {e}") + def setup(app): - app.connect('build-finished', build_datasets_html) + app.connect("build-finished", build_datasets_html) + # sphinx_gallery # -------------- diff --git a/em_database/__init__.py b/em_database/__init__.py index adacee5..70fe522 100644 --- a/em_database/__init__.py +++ b/em_database/__init__.py @@ -1,9 +1,9 @@ ### Example datasets ### -import os +from em_database import data from em_database.config import settings from em_database.downloadable_dataset import DownloadableDataset -from em_database._create_stubs import build_docstring -from em_database import data +from em_database.query import filter, list_datasets, search # noqa: A004 + __all__ = [] @@ -17,8 +17,10 @@ def get_data_dir(): Path to the example datasets directory. """ from em_database import config + return config.data_dir() + def set_data_dir(path: str, persist: bool = True): """ Set the directory where example datasets are stored. @@ -35,6 +37,7 @@ def set_data_dir(path: str, persist: bool = True): if persist: settings.save() + def reset_data_dir(): """ Reset the example datasets directory to the default location, clearing any @@ -47,6 +50,7 @@ def get_setting(key: str, default=None): """Read a value from :data:`em_database.settings`.""" return settings.get(key, default) + def set_setting(key: str, value, persist: bool = True): """Set a value in :data:`em_database.settings`, persisting it by default.""" settings[key] = value @@ -66,18 +70,31 @@ def browse(**kwargs): ``display(em_database)`` renders the same browser. """ from em_database.widget import browse as _browse + return _browse(**kwargs) -__all__ = ['get_data_dir', 'set_data_dir', 'reset_data_dir', - 'get_setting', 'set_setting', 'settings', 'browse', "data"] +__all__ = [ + "list_datasets", + "search", + "filter", + "get_data_dir", + "set_data_dir", + "reset_data_dir", + "get_setting", + "set_setting", + "settings", + "browse", + "data", + "DownloadableDataset", +] # Let ``display(em_database)`` render the browser. Reassigning the module's # __class__ to a ModuleType subclass is a supported pattern (see PEP 562) and is # what lets the package itself carry a rich Jupyter repr. -import sys as _sys -from types import ModuleType as _ModuleType +import sys as _sys # noqa: E402 +from types import ModuleType as _ModuleType # noqa: E402 class _EmDatabaseModule(_ModuleType): @@ -92,7 +109,7 @@ def _repr_mimebundle_(self, include=None, exclude=None, **kwargs): "em_database.browse()." ) } - return widget._repr_mimebundle_(include=include, exclude=exclude, **kwargs) + return widget._repr_mimebundle_(**kwargs) -_sys.modules[__name__].__class__ = _EmDatabaseModule \ No newline at end of file +_sys.modules[__name__].__class__ = _EmDatabaseModule diff --git a/em_database/_build_docs.py b/em_database/_build_docs.py index 843ea4f..e4a0366 100644 --- a/em_database/_build_docs.py +++ b/em_database/_build_docs.py @@ -1,29 +1,36 @@ import json -import yaml -from pathlib import Path from collections import defaultdict +from pathlib import Path + +import yaml + +from em_database.metadata import NON_DATASET_FILES, load_vendors def parse_datasets(yaml_dir): """Parse all YAML files and organize by technique.""" datasets_by_technique = defaultdict(list) - for yaml_file in Path(yaml_dir).glob("*.yaml"): - with open(yaml_file, 'r') as f: + for yaml_file in sorted(Path(yaml_dir).glob("*.yaml")): + if yaml_file.name in NON_DATASET_FILES: + continue + with open(yaml_file, "r") as f: data = yaml.safe_load(f) for name, info in data.items(): - technique = info.get('technique', 'Unknown') - datasets_by_technique[technique].append({ - 'name': name, - 'description': info.get('description', ''), - 'tags': info.get('tags', []), - 'source': info.get('source', ''), - 'file': info.get('file', ''), - 'license': info.get('license', ''), - 'detector': info.get('detector', 'Unknown'), - 'detector_manufacturer': info.get('detector_manufacturer', 'Unknown') - }) + technique = info.get("technique", "Unknown") + datasets_by_technique[technique].append( + { + "name": name, + "description": info.get("description", ""), + "tags": info.get("tags", []), + "source": info.get("source", ""), + "file": info.get("file", ""), + "license": info.get("license", ""), + "detector": info.get("detector", "Unknown"), + "detector_manufacturer": info.get("detector_manufacturer", "Unknown"), + } + ) return dict(datasets_by_technique) @@ -39,10 +46,10 @@ def generate_html_table(datasets_by_technique): tags = set() detectors = {} for dataset in datasets: - tags.update(dataset['tags']) - all_tags.update(dataset['tags']) - manufacturer = dataset.get('detector_manufacturer', 'Unknown') - detector = dataset.get('detector', 'Unknown') + tags.update(dataset["tags"]) + all_tags.update(dataset["tags"]) + manufacturer = dataset.get("detector_manufacturer", "Unknown") + detector = dataset.get("detector", "Unknown") if manufacturer not in detectors: detectors[manufacturer] = set() @@ -57,58 +64,58 @@ def generate_html_table(datasets_by_technique): all_detectors = {m: sorted(d) for m, d in all_detectors.items()} - technique_tags_json = __import__('json').dumps(technique_tags) - technique_detectors_json = __import__('json').dumps(technique_detectors) + technique_tags_json = __import__("json").dumps(technique_tags) + technique_detectors_json = __import__("json").dumps(technique_detectors) all_tags_sorted = sorted(all_tags) - all_detectors_json = __import__('json').dumps(all_detectors) + all_detectors_json = __import__("json").dumps(all_detectors) - html = f""" + html = """ @@ -200,18 +207,18 @@ def generate_html_table(datasets_by_technique): for technique in sorted(datasets_by_technique.keys()): for dataset in datasets_by_technique[technique]: - tags_str = ', '.join(dataset['tags']) - manufacturer = dataset.get('detector_manufacturer', 'Unknown') - detector = dataset.get('detector', 'Unknown') + tags_str = ", ".join(dataset["tags"]) + manufacturer = dataset.get("detector_manufacturer", "Unknown") + detector = dataset.get("detector", "Unknown") detector_full = f"{manufacturer} - {detector}" html += f""" {technique} - {dataset['name']} - {dataset['description']} + {dataset["name"]} + {dataset["description"]} {tags_str} {detector_full} - {dataset['file']} - {dataset['license']} + {dataset["file"]} + {dataset["license"]} """ @@ -220,7 +227,7 @@ def generate_html_table(datasets_by_technique): " + "const TABS = " + json.dumps(tabs) + ";\n" + _DOCS_BROWSER_JS + "\n" ) @@ -793,12 +806,11 @@ def generate_browser_html() -> str: "\n" '\n' '\n' - "EM Datasets\n\n\n" - '
\n' - + _browser_script(payload, tabs) + "\n\n" + + _BROWSER_OVERRIDES + + "\n\n\n" + '
\n' + _browser_script(payload, tabs) + "\n\n" ) @@ -808,17 +820,21 @@ def generate_landing_html() -> str: body = ( '
' '
' - '

EM-Database

' - '

A curated, citable collection of electron microscopy datasets — ' - 'a couple of lines of Python from your analysis. Search below, then copy the ' - 'snippet to load one with em_database.data.<Name>().

' - '
' + "

EM-Database

" + "

A curated, citable collection of electron microscopy datasets — " + "a couple of lines of Python from your analysis. Search below, then copy the " + "snippet to load one with em_database.data.<Name>().

" + "" '
' - '
' + "" + ) + return _app_page( + "EM-Database", + body, + active="", + extra_css=_BROWSER_OVERRIDES, + scripts=_browser_script(payload, tabs), ) - return _app_page("EM-Database", body, active="", - extra_css=_BROWSER_OVERRIDES, - scripts=_browser_script(payload, tabs)) def generate_all_data_html() -> str: @@ -828,22 +844,26 @@ def generate_all_data_html() -> str: '
' '
' '

All Data

' - '

Every dataset in the collection. Search across names, techniques, ' - 'authors, detectors and tags; filter by technique with the tabs.

' - '
' + "

Every dataset in the collection. Search across names, techniques, " + "authors, detectors and tags; filter by technique with the tabs.

" + "" '
' - '
' + "" + ) + return _app_page( + "All Data · EM-Database", + body, + active="All Data", + extra_css=_BROWSER_OVERRIDES + "\n.emdb-body { height: 620px; }\n", + scripts=_browser_script(payload, tabs), ) - return _app_page("All Data · EM-Database", body, active="All Data", - extra_css=_BROWSER_OVERRIDES + "\n.emdb-body { height: 620px; }\n", - scripts=_browser_script(payload, tabs)) # -- Add Dataset page -------------------------------------------------------- -_MANUFACTURERS = ("Gatan", "Thermo Fisher Scientific", "Direct Electron", - "Dectris", "Quantum Detectors", "TVIPS", "Other") -_VENDORS = ("Thermo Fisher Scientific", "JEOL", "Hitachi", "Zeiss", "Other") +_VENDOR_LISTS = load_vendors() +_MANUFACTURERS = tuple(_VENDOR_LISTS["detector_manufacturer"]) +_VENDORS = tuple(_VENDOR_LISTS["microscope_vendor"]) _TECHNIQUES = ("4D-STEM", "EELS", "EDS", "EBSD", "STEM", "In-situ TEM", "Cryo-EM", "Other") # Owner/repo the prefilled "create new file" PR link targets. @@ -857,8 +877,20 @@ def _text_field(fid, label, required=False, placeholder="", hint="", full=False) hn = '
' + _esc(hint) + "
" if hint else "" style = ' style="grid-column:1/-1"' if full else "" return ( - '
" - + hn + '" + '
" + + hn + + '" '
' ) @@ -868,8 +900,41 @@ def _select_field(fid, label, options, hint=""): opts = '' opts += "".join('" for o in options) return ( - '
" - + hn + '" + '
" + + hn + + '" + '
' + ) + + +def _datalist_field(fid, label, options, placeholder="", hint=""): + """A free-text field with suggestions - the open-string vendor lists.""" + hn = '
' + _esc(hint) + "
" if hint else "" + ph = ' placeholder="' + _esc(placeholder) + '"' if placeholder else "" + opts = "".join('
' + + _text_field( + "f-source", + "Source URL", + required=True, + placeholder="https://zenodo.org/records/15490547/files", + hint="Direct download base (no file name).", + ) + + _text_field( + "f-file", + "File", + required=True, + placeholder="smallPtychography.hspy", + hint="The file name at that source.", + ) + + _text_field( + "f-checksum", + "Checksum", + placeholder="md5:df9376d5c020a23f0f7f51cfe79f303f", + hint="md5:<32 hex chars>", + ) + + _text_field( + "f-size_bytes", + "Size (bytes)", + placeholder="1104287335", + hint="The file's Content-Length, in bytes.", + ) + + _datalist_field( + "f-detector_manufacturer", + "Detector Manufacturer", + _MANUFACTURERS, + placeholder="Direct Electron", + ) + _text_field("f-detector", "Detector", placeholder="CeleritasXS") - + _select_field("f-microscope_vendor", "Microscope Vendor", _VENDORS) + + _datalist_field( + "f-microscope_vendor", + "Microscope Vendor", + _VENDORS, + placeholder="Thermo Fisher Scientific", + ) + _text_field("f-microscope_model", "Microscope Model", placeholder="Gen 1 Titan") + _text_field("f-camera_length", "Camera Length", placeholder="e.g. 100 mm") + _text_field("f-voltage", "Voltage", placeholder="200 kV", hint="e.g. 200 kV") + _select_field("f-technique", "Technique", _TECHNIQUES) + _text_field("f-license", "License", placeholder="CC-BY-4.0") + _text_field("f-doi", "DOI", placeholder="10.5281/zenodo.15490547") - + _text_field("f-tags", "Tags", placeholder="Orientation Mapping, Nanocrystals", - hint="Comma-separated.", full=True) + + _text_field( + "f-tags", + "Tags", + placeholder="Orientation Mapping, Nanocrystals", + hint="Comma-separated.", + full=True, + ) ) issue_url = "https://github.com/" + _REPO + "/issues/new?template=new_dataset.yaml" @@ -931,41 +1031,47 @@ def generate_add_dataset_html() -> str: '
' '
' '

Add a Dataset

' - '

Fill in the metadata; the YAML builds live on the right. ' - '“Open a Pull Request” sends you to GitHub with the new file ' - 'pre-filled — commit it to a branch there and GitHub opens the PR.

' - '
' + "

Fill in the metadata; the YAML builds live on the right. " + "“Open a Pull Request” sends you to GitHub with the new file " + "pre-filled — commit it to a branch there and GitHub opens the PR.

" + "
" '
' '
' - '
' + fields + '
' + '
' + fields + "
" '
Authors
' - '
' + _author_row_html() + '
' + '
' + _author_row_html() + "
" '' - '
' + "" '
" '

Requires a GitHub account. The button opens GitHub’s ' - '“create new file” page pre-filled at ' - 'em_database/datasets/<Name>.yaml; if you cannot push to the ' - 'repo, GitHub forks it for you and lets you propose the change. Fields marked ' + "“create new file” page pre-filled at " + "em_database/datasets/<Name>.yaml; if you cannot push to the " + "repo, GitHub forks it for you and lets you propose the change. Fields marked " '* are required.

' - '' - '' - '' + "" + "" + "" ) js = _ADD_DATASET_JS.replace("__REPO__", _REPO).replace("__BRANCH__", _BRANCH) scripts = "" - return _app_page("Add Dataset · EM-Database", body, active="Add Dataset", - extra_css=_FORM_CSS, scripts=scripts) + return _app_page( + "Add Dataset · EM-Database", + body, + active="Add Dataset", + extra_css=_FORM_CSS, + scripts=scripts, + ) _ADD_DATASET_JS = r""" @@ -1022,7 +1128,8 @@ def generate_add_dataset_html() -> str: add("source", val("f-source")); add("checksum", val("f-checksum")); add("file", val("f-file")); - add("data_size", val("f-data_size")); + var bytes = val("f-size_bytes").replace(/[^0-9]/g, ""); + if (bytes) { lines.push(" size_bytes: " + bytes); } add("detector_manufacturer", val("f-detector_manufacturer")); add("detector", val("f-detector")); add("microscope_vendor", val("f-microscope_vendor")); @@ -1145,4 +1252,4 @@ def generate_add_dataset_html() -> str: (out / "all_data.html").write_text(generate_all_data_html(), encoding="utf-8") (out / "add_dataset.html").write_text(generate_add_dataset_html(), encoding="utf-8") (out / "datasets_browser.html").write_text(generate_browser_html(), encoding="utf-8") - print("Wrote preview pages to", out.resolve()) \ No newline at end of file + print("Wrote preview pages to", out.resolve()) diff --git a/em_database/_create_stubs.py b/em_database/_create_stubs.py index fd25fac..9f01647 100644 --- a/em_database/_create_stubs.py +++ b/em_database/_create_stubs.py @@ -1,63 +1,80 @@ -import os -import yaml +"""Generate ``data/__init__.pyi`` so an IDE can complete the dataset classes. + +The classes in :mod:`em_database.data` are built at import time from the YAML, +so nothing static knows their names. The stub is that list, written out. It is +generated from the same files the loader reads, and CI runs ``--check`` to fail +if the committed stub has drifted from them. +""" + +import sys from pathlib import Path -# on start up set data dir if not already set + +import yaml + +from em_database.metadata import dataset_files + +STUB_PATH = Path(__file__).parent / "data" / "__init__.pyi" + + def build_docstring(dataset_dict) -> str: - """ Build a docstring for the dataset from its metadata. """ - doc = f"" + """Build a docstring for the dataset from its metadata.""" + doc = "" if dataset_dict.get("description"): - doc += f"{dataset_dict["description"]}\n\n" + doc += f"{dataset_dict['description']}\n\n" if dataset_dict.get("doi"): - doc += f" DOI: {dataset_dict["doi"]}\n\n" + doc += f" DOI: {dataset_dict['doi']}\n\n" if dataset_dict.get("license"): - doc += f" License: {dataset_dict["license"]}\n\n" + doc += f" License: {dataset_dict['license']}\n\n" doc += " You can download this dataset here:\n" - doc += f" {dataset_dict["source"]}\n\n" + doc += f" {dataset_dict['source']}\n\n" return doc -def generate_pyi_stub(): - """Generate a .pyi stub file for IDE autocomplete support.""" + +def build_pyi_stub() -> str: + """The contents of the ``.pyi`` stub for the current dataset YAML.""" stub_lines = [ "# Auto-generated stub file for em_database", "from em_database.downloadable_dataset import DownloadableDataset", "", ] - # Collect all dataset classes dataset_classes = [] - for root, dirs, files in os.walk(os.path.join(os.path.dirname(__file__), "datasets")): - for file in files: - if file.endswith(".yaml") or file.endswith(".yml"): - dataset_path = os.path.join(root, file) - with open(dataset_path, 'r') as f: - data_dict_yaml = yaml.safe_load(f) - for name in data_dict_yaml: - data_dict = data_dict_yaml[name] - class_name = name.replace(' ', '_').replace('-', '_') - description = build_docstring(data_dict) - - # Add class definition to stub - stub_lines.append(f"class {class_name}(DownloadableDataset):") - stub_lines.append(f' """') - stub_lines.append(f" {name}") - if description: - stub_lines.append(f" ") - stub_lines.append(f" {description}") - stub_lines.append(f' """') - stub_lines.append(" ...") - stub_lines.append("") - - dataset_classes.append(class_name) - - # Add __all__ to stub - stub_lines.append(f"__all__ = __all__ + {dataset_classes }") - - # Write stub file - stub_path = Path(__file__).parent / "data"/"__init__.pyi" - with open(stub_path, 'w') as f: - f.write('\n'.join(stub_lines)) + for dataset_path in dataset_files(): + data_dict_yaml = yaml.safe_load(dataset_path.read_text(encoding="utf-8")) + for name in data_dict_yaml: + data_dict = data_dict_yaml[name] + class_name = name.replace(" ", "_").replace("-", "_") + description = build_docstring(data_dict) + + stub_lines.append(f"class {class_name}(DownloadableDataset):") + stub_lines.append(' """') + stub_lines.append(f" {name}") + if description: + stub_lines.append("") + stub_lines.append(f" {description}") + stub_lines.append(' """') + stub_lines.append(" ...") + stub_lines.append("") + + dataset_classes.append(class_name) + + stub_lines.append(f"__all__ = {dataset_classes}") + return "\n".join(stub_lines) + + +def generate_pyi_stub() -> None: + """Write the stub file.""" + STUB_PATH.write_text(build_pyi_stub(), encoding="utf-8") + if __name__ == "__main__": - generate_pyi_stub() + if "--check" in sys.argv: + if STUB_PATH.read_text(encoding="utf-8") != build_pyi_stub(): + sys.exit( + f"{STUB_PATH} is out of date with the dataset YAML; " + "regenerate it with `python -m em_database._create_stubs`" + ) + else: + generate_pyi_stub() diff --git a/em_database/catalogue.py b/em_database/catalogue.py index 494492e..7da8c50 100644 --- a/em_database/catalogue.py +++ b/em_database/catalogue.py @@ -5,127 +5,128 @@ marked downloaded or not, each carrying the metadata a user hovers to read. It downloads nothing and opens no files, so it is cheap enough to rebuild on every render (the one thing that changes underfoot is which files are on disk, which -is a single ``os.path.exists`` per dataset). +is a single ``Path.exists`` per dataset). """ + from __future__ import annotations import inspect -from typing import Any, Optional +import warnings +from pathlib import Path +from em_database.downloadable_dataset import DownloadableDataset +from em_database.metadata import DatasetMetadata # Techniques in the order the browser should show them - the modalities the # collection is built around first, then anything else alphabetically. TECHNIQUE_ORDER = ("4D-STEM", "EELS", "EDS", "EBSD", "STEM", "In-situ TEM", "Cryo-EM") -def datasets() -> list[tuple[str, Any]]: +def datasets() -> list[tuple[str, DownloadableDataset]]: """``(name, dataset)`` for every dataset ``em_database.data`` exposes. Filtered by ``issubclass`` so the base class and incidental imports in the module namespace stay out; sorted by name for a stable order. """ import em_database.data as data - from em_database.downloadable_dataset import DownloadableDataset - out: list[tuple[str, Any]] = [] + out: list[tuple[str, DownloadableDataset]] = [] for name in getattr(data, "__all__", None) or dir(data): if name.startswith("_"): continue obj = getattr(data, name, None) - if (not inspect.isclass(obj) or obj is DownloadableDataset - or not issubclass(obj, DownloadableDataset)): + if ( + not inspect.isclass(obj) + or obj is DownloadableDataset + or not issubclass(obj, DownloadableDataset) + ): continue try: out.append((name, obj())) - except Exception: - continue + except TypeError as error: + # from_spec rejects a malformed entry. em_database.data validates at + # import so this should be unreachable, but a dataset silently + # missing from the browser is the wrong way to find out otherwise. + warnings.warn(f"skipping dataset {name!r}: {error}", stacklevel=2) return sorted(out, key=lambda kv: kv[0].lower()) -def resolve(name: str): +def resolve(name: str) -> DownloadableDataset | None: """The dataset instance for a catalogue name, or ``None``.""" import em_database.data as data obj = getattr(data, str(name), None) - return obj() if inspect.isclass(obj) else None + if not inspect.isclass(obj) or not issubclass(obj, DownloadableDataset): + return None + return obj() -def _technique(ds) -> str: - md = getattr(ds, "metadata", None) or {} - return str(md.get("technique") or "Other").strip() or "Other" +def _technique(md: DatasetMetadata) -> str: + return (md.technique or "Other").strip() or "Other" def _join(*parts) -> str: return " ".join(str(p).strip() for p in parts if p and str(p).strip()) -def _declared_shape(ds) -> Optional[str]: - """The shape em-database declares in the dataset's YAML, if it does. +def _location(path: Path | None) -> str | None: + """Which data directory a downloaded file came from: "shared" or "user".""" + if path is None: + return None + from em_database import config - Only declared shapes are used - reading it out of a downloaded file would - mean opening the file (and depending on a reader), which the catalogue - deliberately never does. - """ - md = getattr(ds, "metadata", None) or {} - for source in (md, ds): - for attr in ("shape", "data_shape"): - val = source.get(attr) if isinstance(source, dict) else getattr(source, attr, None) - if val: - return val.strip() if isinstance(val, str) else "×".join(str(v) for v in val) - return None + parent = path.resolve().parent + shared = {d.resolve() for d in config.shared_data_dirs()} + return "shared" if parent in shared else "user" -def _authors(md) -> tuple[list[str], list[str]]: - """``(names, affiliations)`` from the dataset's ``authors`` metadata. - - Authors are usually ``{name: {affiliation: ...}}``; also tolerate a plain - list of names. - """ - authors = md.get("authors") - if isinstance(authors, dict): - names = list(authors.keys()) - affiliations = [] - for value in authors.values(): - if isinstance(value, dict) and value.get("affiliation"): - affiliations.append(str(value["affiliation"])) - return names, affiliations - return [str(a) for a in (authors or [])], [] - - -def entry(name: str, ds) -> dict: +def entry(name: str, ds: DownloadableDataset) -> dict: """One catalogue row - everything the browser draws for a dataset.""" - md = getattr(ds, "metadata", None) or {} + md = ds.metadata try: - path = ds.filepath() + found = ds.filepaths() except Exception: - path = None - names, affiliations = _authors(md) + found = [] + path = found[0] if found else None + # The copy in the user's own directory, which may sit behind a shared one in + # the search order. It is the only copy delete() will touch, so the widgets + # need it to know whether there is anything to offer deleting. + user_path = next((p for p in found if _location(p) == "user"), None) row = { "name": name, - "technique": _technique(ds), - "size": str(getattr(ds, "data_size", "") or ""), - "shape": _declared_shape(ds), - "downloaded": bool(path), - "path": path or "", - "description": str(getattr(ds, "description", "") or ""), - "detector": _join(getattr(ds, "detector_manufacturer", ""), getattr(ds, "detector", "")), - "microscope": _join(md.get("microscope_vendor"), md.get("microscope_model")), - "voltage": str(md.get("voltage") or ""), - "tags": [str(t) for t in (md.get("tags") or [])], - "authors": names, - "license": str(getattr(ds, "license", "") or ""), - "doi": str(getattr(ds, "doi", "") or ""), - "source": str(getattr(ds, "source", "") or ""), - "file": str(getattr(ds, "file", "") or ""), + "technique": _technique(md), + "size": md.size, + "downloaded": path is not None, + "location": _location(path), + "path": str(path) if path else "", + "user_path": str(user_path) if user_path else "", + "description": md.description, + "detector": _join(md.detector_manufacturer, md.detector), + "microscope": _join(md.microscope_vendor, md.microscope_model), + "voltage": md.voltage or "", + "tags": list(md.tags), + "authors": list(md.authors), + "license": md.license or "", + "doi": md.doi or "", + "source": md.source, + "file": md.file, } # One lowercased blob the search box matches against, so a query like # "Carter Francis" (an author) or "Direct Electron" (an affiliation) finds # every dataset it touches - not just the name. searchable = [ - name, row["technique"], row["description"], row["detector"], - row["microscope"], row["voltage"], row["license"], row["doi"], - row["file"], row["shape"] or "", " ".join(row["tags"]), - " ".join(names), " ".join(affiliations), + name, + row["technique"], + row["description"], + row["detector"], + row["microscope"], + row["voltage"], + row["license"], + row["doi"], + row["file"], + " ".join(row["tags"]), + " ".join(row["authors"]), + " ".join(a.affiliation for a in md.authors.values()), ] row["search"] = " ".join(str(s) for s in searchable if s).lower() return row diff --git a/em_database/config.py b/em_database/config.py index 1fa7bb3..2a03346 100644 --- a/em_database/config.py +++ b/em_database/config.py @@ -24,6 +24,7 @@ environment variable (an ``os.pathsep``-separated list), or a ``shared_data_dirs`` list in the user's settings. """ + from __future__ import annotations import os @@ -32,19 +33,20 @@ import yaml -def _default_data_dir() -> str: - return os.path.join(os.path.expanduser("~"), "em_database") +def _default_data_dir() -> Path: + return Path.home() / "em_database" -#: Built-in defaults - the fallback when nothing is configured. +#: Built-in defaults - the fallback when nothing is configured. Values here are +#: the serialized (string) form, so they compare equal to what the YAML holds. DEFAULTS: dict = { - "data_dir": _default_data_dir(), + "data_dir": str(_default_data_dir()), } def config_dir() -> Path: """The ``~/.em_database`` folder that holds the settings file.""" - return Path(os.path.expanduser("~")) / ".em_database" + return Path.home() / ".em_database" def config_path() -> Path: @@ -155,6 +157,7 @@ def reset(self, key: str | None = None) -> None: def widget(self): """Return an interactive widget for editing the settings (Jupyter).""" from em_database.widget import settings_widget + return settings_widget() def _repr_mimebundle_(self, **kwargs): @@ -162,6 +165,7 @@ def _repr_mimebundle_(self, **kwargs): plain dict repr if anywidget is not installed).""" try: from em_database.widget import settings_widget + widget = settings_widget() except Exception: return {"text/plain": repr(dict(self))} @@ -173,12 +177,12 @@ def _repr_mimebundle_(self, **kwargs): _seed(settings) -def data_dir() -> str: +def data_dir() -> Path: """The user's data directory - where downloads are written.""" - return str(settings.get("data_dir") or _default_data_dir()) + return Path(settings.get("data_dir") or _default_data_dir()) -def shared_data_dirs() -> list[str]: +def shared_data_dirs() -> list[Path]: """System-wide / shared data locations, checked before the user's dir. Order: ``EM_DATABASE_SHARED_DIR`` (an ``os.pathsep`` list), then the user's @@ -195,15 +199,15 @@ def shared_data_dirs() -> list[str]: dirs.append(str(system["data_dir"])) dirs += [str(d) for d in (system.get("shared_data_dirs") or [])] seen: set[str] = set() - unique: list[str] = [] + unique: list[Path] = [] for d in dirs: if d not in seen: seen.add(d) - unique.append(d) + unique.append(Path(d)) return unique -def data_search_dirs() -> list[str]: +def data_search_dirs() -> list[Path]: """Everywhere to look for an existing dataset: shared/system dirs first, then the user's data directory.""" dirs = shared_data_dirs() diff --git a/em_database/data/__init__.py b/em_database/data/__init__.py index f0e5ed5..801eb1e 100644 --- a/em_database/data/__init__.py +++ b/em_database/data/__init__.py @@ -1,36 +1,26 @@ -""" Auto-generated dataset classes from YAML Files for downloading data.""" -import os -from pathlib import Path +"""Auto-generated dataset classes from YAML Files for downloading data.""" import yaml -from em_database.downloadable_dataset import DownloadableDataset + from em_database._create_stubs import build_docstring +from em_database.downloadable_dataset import DownloadableDataset +from em_database.metadata import DatasetMetadata, dataset_files -# Map all the datasets in the "datasets" folder -# recursively travel down __all__ = [] -datasets_path = Path(__file__).parent.parent / "datasets" -for root, dirs, files in os.walk(datasets_path): - for file in files: - if file.endswith(".yaml") or file.endswith(".yml"): - dataset_path = os.path.join(root, file) - with open(dataset_path, 'r') as f: - data_dict_yaml = yaml.safe_load(f) - for name in data_dict_yaml: - class_name = name.replace(' ', '_').replace('-', '_') - data_dict = data_dict_yaml[name] - def _make_init(data): - def __init__(self): - super(self.__class__, self).__init__(**data) - - return __init__ - - - _new_class = type(class_name, - (DownloadableDataset,), - {"__init__": _make_init(data_dict), - "__doc__": build_docstring(data_dict)}) - - # Add to module globals and __all__ - globals()[class_name] = _new_class - __all__.append(class_name) \ No newline at end of file +for dataset_path in dataset_files(): + data_dict_yaml = yaml.safe_load(dataset_path.read_text(encoding="utf-8")) + for name in data_dict_yaml: + class_name = name.replace(" ", "_").replace("-", "_") + data_dict = data_dict_yaml[name] + _new_class = type( + class_name, + (DownloadableDataset,), + { + "_spec": data_dict, + "_origin": dataset_path, + "_metadata": DatasetMetadata.from_spec(data_dict, dataset_path), + "__doc__": build_docstring(data_dict), + }, + ) + globals()[class_name] = _new_class + __all__.append(class_name) diff --git a/em_database/data/__init__.pyi b/em_database/data/__init__.pyi index c28ba0e..379d32d 100644 --- a/em_database/data/__init__.pyi +++ b/em_database/data/__init__.pyi @@ -4,7 +4,7 @@ from em_database.downloadable_dataset import DownloadableDataset class AlNanocrystals(DownloadableDataset): """ AlNanocrystals - + A 4D STEM dataset of Al nanocrystals on a carbon support. License: CC-BY-4.0 @@ -19,7 +19,7 @@ class AlNanocrystals(DownloadableDataset): class AmorphousFilm4nm4DSTEM(DownloadableDataset): """ AmorphousFilm4nm4DSTEM - + A 4D-STEM dataset of a 4 nm amorphous thin film acquired with a 2.5 mrad probe on a Direct Electron CeleritasXS at 49000 fps. 256 x 256 probe positions (the central quarter of a 1024 x 1024 scan) of 128 x 128 pixel diffraction patterns. Both real and reciprocal space are calibrated - 0.12325 nm per scan step and 0.12453 1/nm per detector pixel, centred on the direct beam. Suitable for fluctuation electron microscopy and angular correlation analysis. Gain- and dark-corrected intensities were divided by 2 and rounded to uint16; multiply by 2 to recover ADU (the detector records 300 ADU per electron). DOI: 10.5281/zenodo.21632101 @@ -36,7 +36,7 @@ class AmorphousFilm4nm4DSTEM(DownloadableDataset): class ApoferritinApollo15eps(DownloadableDataset): """ ApoferritinApollo15eps - + A single cryo-EM movie of apoferritin from one stage position, collected at a 15 e-/pix/s dose rate on a Direct Electron Apollo. 76 unaligned, dark-subtracted counted super-resolution frames of 8192 x 8192 pixels at 0.2995 Angstrom per pixel, totalling 56.96 e-/Angstrom^2 (0.7495 e-/Angstrom^2 per frame). The super-resolution gain reference needed for frame correction is embedded in the file at metadata.Acquisition_instrument.TEM.Detector.gain_reference. Repackaged from EMPIAR-11254; see Peng et al., J Struct Biol X 7 (2022) 100080. DOI: 10.5281/zenodo.21632101 @@ -53,7 +53,7 @@ class ApoferritinApollo15eps(DownloadableDataset): class BilayerWS2(DownloadableDataset): """ BilayerWS2 - + small 4-D STEM dataset of a bilayer WS2. Each Diffraction pattern is only 8x8 pixels so the dataset is quite small although for simple non iterative ptychography 8x8 pixels should be sufficient. License: CC-BY-4.0 @@ -68,7 +68,7 @@ class BilayerWS2(DownloadableDataset): class CuZnEELSMapping(DownloadableDataset): """ CuZnEELSMapping - + An EELS spectrum image of copper and zinc oxide deposited on carbon nanotubes, used by the eXSpy elemental mapping tutorial. 40 x 50 probe positions at 0.9214 nm with 162 energy channels covering 700-1988 eV at 8 eV dispersion, which resolves the Cu-L2,3 (~931 eV) and Zn-L2,3 (~1020 eV) edges. The Zn:Cu ratio is 3:1 and roughly 80 wt% of the sample is carbon, so the edges sit on a large plasmon background - a good test case for model-based background removal and overlapping-edge quantification. The simultaneously acquired survey image is available as CuZnHAADF. Note - the Sample.description field in the file reads "Ta2O5 25% TiO2 CSIRO 400C", which is stale metadata carried over from an unrelated acquisition. License: Unspecified @@ -83,7 +83,7 @@ class CuZnEELSMapping(DownloadableDataset): class CuZnHAADF(DownloadableDataset): """ CuZnHAADF - + The HAADF survey image acquired simultaneously with the CuZnEELSMapping spectrum image - copper and zinc oxide on carbon nanotubes. 40 x 50 pixels at 0.9214 nm, uint16, on a JEOL ARM200F at 200 kV and 600000x. Pairs with CuZnEELSMapping for correlating elemental maps against the survey. Note - the Sample.description field in the file reads "Ta2O5 25% TiO2 CSIRO 400C", which is stale metadata carried over from an unrelated acquisition. License: Unspecified @@ -98,7 +98,7 @@ class CuZnHAADF(DownloadableDataset): class FeAlStripes(DownloadableDataset): """ FeAlStripes - + A 4D STEM dataset with FeAl stripes exhibiting magnetic stripes. The dataset can be used to study the correlation between structural and magnetic properties. License: CC-BY-4.0 @@ -113,7 +113,7 @@ class FeAlStripes(DownloadableDataset): class HREBSDStrainPatterns(DownloadableDataset): """ HREBSDStrainPatterns - + High-resolution EBSD patterns collected on a Direct Electron DE-Meridian, centred on a deformed region suitable for cross-correlation strain analysis. 32 x 32 probe positions at a 25 nm step, each pattern the sum of 256 counted, dark- and gain-corrected frames, binned 2 x 2 from 2048 x 2048 to 1024 x 1024 pixels. The pattern-plane geometry (detector distance, pattern centre, sample tilt) and the accelerating voltage were not recorded with the raw data, so the pattern axes are in detector pixels. DOI: 10.5281/zenodo.21632101 @@ -130,25 +130,8 @@ class HREBSDStrainPatterns(DownloadableDataset): class InSituElectrochemGrowth(DownloadableDataset): """ InSituElectrochemGrowth - - An in-situ electrochemistry TEM movie showing growth in a liquid cell, recorded at 300 kV on a Direct Electron DE-Artemis in hardware counting mode. 245 frames of 4096 x 4096 pixels - every 4th frame of a 977 frame movie - with a calibrated 0.45448 nm pixel size and a 0.26208 s interval between saved frames (1.86 x 1.86 micron field of view, 64 s of elapsed time). - - DOI: 10.5281/zenodo.21632101 - - License: CC-BY-4.0 - You can download this dataset here: - https://zenodo.org/records/21632101/files - - - """ - ... - -class LayeredCuNb4DSTEM(DownloadableDataset): - """ - LayeredCuNb4DSTEM - - A 4D-STEM dataset of a layered Cu/Nb nanolaminate, acquired with a nearly parallel 1.58 mrad probe on a Direct Electron CeleritasXS. 128 x 128 probe positions (the central quarter of a 512 x 512 raster scan) of 256 x 256 pixel diffraction patterns, each the sum of 32 camera frames at 25000 fps. Reciprocal space is calibrated at 0.0078768 1/Angstrom per pixel and centred on the direct beam; the detector half-width is 1.008 1/Angstrom (25.3 mrad at 200 kV). The real-space scan step was not recorded by the scan controller, so the navigation axes are in pixels. Gain- and dark-corrected intensities were divided by 2 and rounded to uint16; multiply by 2 to recover ADU. + An in-situ electrochemistry TEM movie showing growth in a liquid cell, recorded at 300 kV on a Direct Electron DE-Artemis in hardware counting mode. 245 frames of 4096 x 4096 pixels - every 4th frame of a 977 frame movie - with a calibrated 0.45448 nm pixel size and a 0.26208 s interval between saved frames (1.86 x 1.86 micron field of view, 64 s of elapsed time). DOI: 10.5281/zenodo.21632101 @@ -164,7 +147,7 @@ class LayeredCuNb4DSTEM(DownloadableDataset): class LSMOLineScan(DownloadableDataset): """ LSMOLineScan - + A core-loss EELS line scan through a La(0.7)Sr(0.3)MnO3 thin film in which part of the film was deliberately given a very long electron beam exposure, inducing oxygen vacancies. Used by the eXSpy fine structure tutorial - the O-K and Mn-L2,3 fine structure changes measurably between the damaged and undamaged regions. 40 probe positions at 3.219 nm with 586 energy channels covering 428.5-721 eV at 0.5 eV dispersion. Acquired on a JEOL ARM200cF with a Gatan Quantum ER in DualEELS mode at 80 kV, 27.42 mrad convergence and 33.19 mrad collection angle. The matching low-loss spectrum is LSMOLineScanLowLoss. License: Unspecified @@ -179,7 +162,7 @@ class LSMOLineScan(DownloadableDataset): class LSMOLineScanLowLoss(DownloadableDataset): """ LSMOLineScanLowLoss - + The low-loss half of the DualEELS pair for LSMOLineScan - a La(0.7)Sr(0.3)MnO3 thin film line scan through a beam-damaged, oxygen-deficient region. 40 probe positions at 3.219 nm with 1024 energy channels covering -50 to 461.5 eV at 0.5 eV dispersion, containing the zero-loss peak and plasmon region. Use it with the core-loss spectrum for relative thickness mapping and Fourier-ratio deconvolution before fine structure analysis. Acquired on a JEOL ARM200cF with a Gatan Quantum ER at 80 kV. License: Unspecified @@ -194,7 +177,7 @@ class LSMOLineScanLowLoss(DownloadableDataset): class LSMOSTOLineScan(DownloadableDataset): """ LSMOSTOLineScan - + A core-loss EELS line scan across a La(0.7)Sr(0.3)MnO3 thin film grown on SrTiO3, used by the eXSpy perovskite oxide analysis tutorial. 10 probe positions at 3.152 nm with 512 energy channels covering 395-906 eV at 1 eV dispersion, which contains the Ti-L2,3, O-K, Mn-L2,3 and La-M4,5 edges - so a single line scan crosses the interface and shows the Ti signal give way to La and Mn. Acquired on a JEOL ARM200cF with a Gatan Quantum ER in DualEELS mode at 200 kV, 27.1 mrad convergence and 33.1 mrad collection angle. The matching low-loss spectrum needed for thickness correction and Fourier-ratio deconvolution is LSMOSTOLineScanLowLoss. Binned from the original acquisition to keep the file small. License: Unspecified @@ -209,7 +192,7 @@ class LSMOSTOLineScan(DownloadableDataset): class LSMOSTOLineScanLowLoss(DownloadableDataset): """ LSMOSTOLineScanLowLoss - + The low-loss half of the DualEELS pair for LSMOSTOLineScan - a La(0.7)Sr(0.3)MnO3 on SrTiO3 thin film line scan. 10 probe positions at 3.152 nm with 512 energy channels covering -50 to 461 eV at 1 eV dispersion, so it contains the zero-loss peak and the plasmon region. Use it with the core-loss spectrum for relative thickness mapping and Fourier-ratio deconvolution. Acquired on a JEOL ARM200cF with a Gatan Quantum ER at 200 kV; the low-loss dwell time is 9.44e-05 s against 0.4999 s for the core loss. License: Unspecified @@ -218,13 +201,30 @@ class LSMOSTOLineScanLowLoss(DownloadableDataset): https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets + """ + ... + +class LayeredCuNb4DSTEM(DownloadableDataset): + """ + LayeredCuNb4DSTEM + + A 4D-STEM dataset of a layered Cu/Nb nanolaminate, acquired with a nearly parallel 1.58 mrad probe on a Direct Electron CeleritasXS. 128 x 128 probe positions (the central quarter of a 512 x 512 raster scan) of 256 x 256 pixel diffraction patterns, each the sum of 32 camera frames at 25000 fps. Reciprocal space is calibrated at 0.0078768 1/Angstrom per pixel and centred on the direct beam; the detector half-width is 1.008 1/Angstrom (25.3 mrad at 200 kV). The real-space scan step was not recorded by the scan controller, so the navigation axes are in pixels. Gain- and dark-corrected intensities were divided by 2 and rounded to uint16; multiply by 2 to recover ADU. + + DOI: 10.5281/zenodo.21632101 + + License: CC-BY-4.0 + + You can download this dataset here: + https://zenodo.org/records/21632101/files + + """ ... class MgONanoCrystals(DownloadableDataset): """ MgONanoCrystals - + A 4D STEM dataset of various MgO nanocrystals License: CC-BY-4.0 @@ -239,7 +239,7 @@ class MgONanoCrystals(DownloadableDataset): class NiEBSDLarge(DownloadableDataset): """ NiEBSDLarge - + 4125 EBSD patterns in a (55, 75) navigation shape of (60, 60) pixels from nickel, acquired on a NORDIF UF-1100 detector License: CC-BY-4.0 @@ -254,7 +254,7 @@ class NiEBSDLarge(DownloadableDataset): class PdCuSiCrystallization(DownloadableDataset): """ PdCuSiCrystallization - + A time resolved 4D-STEM series following the crystallization of a PdCuSi metallic glass, acquired on a Direct Electron CeleritasXS at 40000 fps with a 25 microsecond dwell time. 400 sequential scans of 47 x 39 probe positions (a 23.5 x 19.5 nm region cropped from a 256 x 256 scan) of 128 x 128 pixel diffraction patterns. Both real and reciprocal space are calibrated - 0.5 nm per scan step and 0.11 1/nm per detector pixel, centred on the direct beam. Successive scans are 1.6384 s apart and span 655.36 s of elapsed time, although the time axis is stored with a nm unit label. 24 GB of uint16 data uncompressed, chunked one time step at a time - the dataset used in the pyxem large data and lazy processing demo. DOI: 10.5281/zenodo.15490547 @@ -271,7 +271,7 @@ class PdCuSiCrystallization(DownloadableDataset): class PdNiPGlass(DownloadableDataset): """ PdNiPGlass - + A 4D STEM dataset of PdNiP metallic glass thin film. License: CC-BY-4.0 @@ -286,7 +286,7 @@ class PdNiPGlass(DownloadableDataset): class SPEDAg(DownloadableDataset): """ SPEDAg - + A 4D STEM dataset of polycrystalline Ag including twins and grain boundaries. License: CC-BY-4.0 @@ -301,7 +301,7 @@ class SPEDAg(DownloadableDataset): class ZrNbPrecipitate(DownloadableDataset): """ ZrNbPrecipitate - + A 4D STEM dataset of ZrNb precipitate in ZrNb alloy. License: CC-BY-4.0 @@ -313,4 +313,4 @@ class ZrNbPrecipitate(DownloadableDataset): """ ... -__all__ = __all__ + ['AlNanocrystals', 'AmorphousFilm4nm4DSTEM', 'ApoferritinApollo15eps', 'BilayerWS2', 'CuZnEELSMapping', 'CuZnHAADF', 'FeAlStripes', 'HREBSDStrainPatterns', 'InSituElectrochemGrowth', 'LayeredCuNb4DSTEM', 'LSMOLineScan', 'LSMOLineScanLowLoss', 'LSMOSTOLineScan', 'LSMOSTOLineScanLowLoss', 'MgONanoCrystals', 'NiEBSDLarge', 'PdCuSiCrystallization', 'PdNiPGlass', 'SPEDAg', 'ZrNbPrecipitate'] \ No newline at end of file +__all__ = ['AlNanocrystals', 'AmorphousFilm4nm4DSTEM', 'ApoferritinApollo15eps', 'BilayerWS2', 'CuZnEELSMapping', 'CuZnHAADF', 'FeAlStripes', 'HREBSDStrainPatterns', 'InSituElectrochemGrowth', 'LSMOLineScan', 'LSMOLineScanLowLoss', 'LSMOSTOLineScan', 'LSMOSTOLineScanLowLoss', 'LayeredCuNb4DSTEM', 'MgONanoCrystals', 'NiEBSDLarge', 'PdCuSiCrystallization', 'PdNiPGlass', 'SPEDAg', 'ZrNbPrecipitate'] \ No newline at end of file diff --git a/em_database/datasets/AlNanocrystals.yaml b/em_database/datasets/AlNanocrystals.yaml index 9595f9b..8e85b74 100644 --- a/em_database/datasets/AlNanocrystals.yaml +++ b/em_database/datasets/AlNanocrystals.yaml @@ -4,7 +4,7 @@ AlNanocrystals: source: https://zenodo.org/records/15490547/files checksum: md5:25901cc272da7d69859f520985230e98 file: AlNanocrystalsSmaller.zspy - data_size: 1.4 GB + size_bytes: 1381831542 detector_manufacturer: Direct Electron detector: CeleritasXS microscope_vendor: Thermo Fisher Scientific diff --git a/em_database/datasets/AmorphousFilm4nm4DSTEM.yaml b/em_database/datasets/AmorphousFilm4nm4DSTEM.yaml index 2469c89..28dea40 100644 --- a/em_database/datasets/AmorphousFilm4nm4DSTEM.yaml +++ b/em_database/datasets/AmorphousFilm4nm4DSTEM.yaml @@ -11,7 +11,7 @@ AmorphousFilm4nm4DSTEM: source: https://zenodo.org/records/21632101/files checksum: md5:eaa809326d134e73b2c96e6ad4a85b6a file: AmorphousFilm4nm4DSTEM.zspy - data_size: 1.10 GB + size_bytes: 1104287335 detector_manufacturer: Direct Electron detector: CeleritasXS license: CC-BY-4.0 diff --git a/em_database/datasets/ApoferritinApollo15eps.yaml b/em_database/datasets/ApoferritinApollo15eps.yaml index f055270..bf001b9 100644 --- a/em_database/datasets/ApoferritinApollo15eps.yaml +++ b/em_database/datasets/ApoferritinApollo15eps.yaml @@ -10,7 +10,7 @@ ApoferritinApollo15eps: source: https://zenodo.org/records/21632101/files checksum: md5:ce63b6d28567b6a74b8c12712b6dab21 file: ApoferritinApollo15eps.zspy - data_size: 483.0 MB + size_bytes: 483010772 detector_manufacturer: Direct Electron detector: Apollo license: CC0-1.0 diff --git a/em_database/datasets/BilayerWS2.yaml b/em_database/datasets/BilayerWS2.yaml index 512cfc9..c2f4884 100644 --- a/em_database/datasets/BilayerWS2.yaml +++ b/em_database/datasets/BilayerWS2.yaml @@ -4,7 +4,7 @@ BilayerWS2: source: https://zenodo.org/records/15490547/files checksum: md5:df9376d5c020a23f0f7f51cfe79f303f file: smallPtychography.hspy - data_size: 12.5 MB + size_bytes: 12492298 detector_manufacturer: Direct Electron detector: CeleritasXS microscope_vendor: Thermo Fisher Scientific diff --git a/em_database/datasets/CuZnEELSMapping.yaml b/em_database/datasets/CuZnEELSMapping.yaml index 58329a0..bfa8a4d 100644 --- a/em_database/datasets/CuZnEELSMapping.yaml +++ b/em_database/datasets/CuZnEELSMapping.yaml @@ -12,7 +12,7 @@ CuZnEELSMapping: source: https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets checksum: md5:fe46eb743e213eef7179538242b1f121 file: CuZn_EELS_mapping_tutorial.hspy - data_size: 466.1 kB + size_bytes: 466089 detector_manufacturer: Gatan detector: GIF Quantum ER microscope_vendor: JEOL diff --git a/em_database/datasets/CuZnHAADF.yaml b/em_database/datasets/CuZnHAADF.yaml index 0d6dc92..5e29d27 100644 --- a/em_database/datasets/CuZnHAADF.yaml +++ b/em_database/datasets/CuZnHAADF.yaml @@ -9,7 +9,7 @@ CuZnHAADF: source: https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets checksum: md5:b328b4a3c78828a6c704db332fc1e0dc file: CuZn_HAADF.hspy - data_size: 34.0 kB + size_bytes: 34043 detector_manufacturer: Gatan microscope_vendor: JEOL microscope_model: ARM200F diff --git a/em_database/datasets/FeAlStripes.yaml b/em_database/datasets/FeAlStripes.yaml index 8e7a3ef..a6d345a 100644 --- a/em_database/datasets/FeAlStripes.yaml +++ b/em_database/datasets/FeAlStripes.yaml @@ -4,7 +4,7 @@ FeAlStripes: source: https://zenodo.org/records/15490547/files checksum: md5:702cb0c8ff75062c0cb23b3722c2f859 file: FeAl_stripes.zspy - data_size: 382.1 MB + size_bytes: 382141850 detector_manufacturer: Quantum Detectors detector: Merlin microscope_vendor: JEOL diff --git a/em_database/datasets/HREBSDStrainPatterns.yaml b/em_database/datasets/HREBSDStrainPatterns.yaml index ac381ff..bce768c 100644 --- a/em_database/datasets/HREBSDStrainPatterns.yaml +++ b/em_database/datasets/HREBSDStrainPatterns.yaml @@ -10,7 +10,7 @@ HREBSDStrainPatterns: source: https://zenodo.org/records/21632101/files checksum: md5:76ad1ba1c0730faf629f03785d49ecd9 file: HREBSDStrainPatterns.zspy - data_size: 1.28 GB + size_bytes: 1280664400 detector_manufacturer: Direct Electron detector: DE-Meridian license: CC-BY-4.0 diff --git a/em_database/datasets/InSituElectrochemGrowth.yaml b/em_database/datasets/InSituElectrochemGrowth.yaml index 0782ba9..898f88f 100644 --- a/em_database/datasets/InSituElectrochemGrowth.yaml +++ b/em_database/datasets/InSituElectrochemGrowth.yaml @@ -8,7 +8,7 @@ InSituElectrochemGrowth: source: https://zenodo.org/records/21632101/files checksum: md5:5f509bbecb7913387d2897bb01f66229 file: InSituElectrochemGrowth.zspy - data_size: 2.44 GB + size_bytes: 2443313730 detector_manufacturer: Direct Electron detector: DE-Artemis voltage: 300 kV diff --git a/em_database/datasets/LSMOLineScan.yaml b/em_database/datasets/LSMOLineScan.yaml index 098695b..f5d484d 100644 --- a/em_database/datasets/LSMOLineScan.yaml +++ b/em_database/datasets/LSMOLineScan.yaml @@ -11,7 +11,7 @@ LSMOLineScan: source: https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets checksum: md5:9cfcf5fbcaac71709b69b79df12faf98 file: LSMO_linescan.hspy - data_size: 91.9 kB + size_bytes: 91885 detector_manufacturer: Gatan detector: Quantum ER microscope_vendor: JEOL diff --git a/em_database/datasets/LSMOLineScanLowLoss.yaml b/em_database/datasets/LSMOLineScanLowLoss.yaml index 742f8f1..6f5b383 100644 --- a/em_database/datasets/LSMOLineScanLowLoss.yaml +++ b/em_database/datasets/LSMOLineScanLowLoss.yaml @@ -10,7 +10,7 @@ LSMOLineScanLowLoss: source: https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets checksum: md5:37881582dd337c5f233befe6b99ed58a file: LSMO_linescan_low_loss.hspy - data_size: 138.8 kB + size_bytes: 138806 detector_manufacturer: Gatan detector: Quantum ER microscope_vendor: JEOL diff --git a/em_database/datasets/LSMOSTOLineScan.yaml b/em_database/datasets/LSMOSTOLineScan.yaml index 94f901d..152e9ae 100644 --- a/em_database/datasets/LSMOSTOLineScan.yaml +++ b/em_database/datasets/LSMOSTOLineScan.yaml @@ -12,7 +12,7 @@ LSMOSTOLineScan: source: https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets checksum: md5:de31f7d7dc222727fac6e5ba92cac825 file: LSMO_STO_linescan.hspy - data_size: 42.5 kB + size_bytes: 42538 detector_manufacturer: Gatan detector: Quantum ER microscope_vendor: JEOL diff --git a/em_database/datasets/LSMOSTOLineScanLowLoss.yaml b/em_database/datasets/LSMOSTOLineScanLowLoss.yaml index 4b8d343..240a78b 100644 --- a/em_database/datasets/LSMOSTOLineScanLowLoss.yaml +++ b/em_database/datasets/LSMOSTOLineScanLowLoss.yaml @@ -10,7 +10,7 @@ LSMOSTOLineScanLowLoss: source: https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets checksum: md5:fd6e922c73e0b1e11229acb5960370cc file: LSMO_STO_linescan_low_loss.hspy - data_size: 42.0 kB + size_bytes: 42038 detector_manufacturer: Gatan detector: Quantum ER microscope_vendor: JEOL diff --git a/em_database/datasets/LayeredCuNb4DSTEM.yaml b/em_database/datasets/LayeredCuNb4DSTEM.yaml index 26b2104..2f78ac2 100644 --- a/em_database/datasets/LayeredCuNb4DSTEM.yaml +++ b/em_database/datasets/LayeredCuNb4DSTEM.yaml @@ -11,7 +11,7 @@ LayeredCuNb4DSTEM: source: https://zenodo.org/records/21632101/files checksum: md5:4507f116a29de495195b0dca652b682e file: LayeredCuNb4DSTEM.zspy - data_size: 720.9 MB + size_bytes: 720938200 detector_manufacturer: Direct Electron detector: CeleritasXS voltage: 200 kV diff --git a/em_database/datasets/MgONanoCrystals.yaml b/em_database/datasets/MgONanoCrystals.yaml index 1edefc9..704a5f3 100644 --- a/em_database/datasets/MgONanoCrystals.yaml +++ b/em_database/datasets/MgONanoCrystals.yaml @@ -4,7 +4,7 @@ MgONanoCrystals: source: https://zenodo.org/records/15490547/files checksum: md5:38a03c23cec147a9cffab9ea349cd15a file: mgo_nano.zspy - data_size: 104.3 MB + size_bytes: 104291721 detector_manufacturer: Allied Vision detector: Stingray microscope_vendor: JEOL diff --git a/em_database/datasets/NiEBSDLarge.yaml b/em_database/datasets/NiEBSDLarge.yaml index 73bddee..7a6473b 100644 --- a/em_database/datasets/NiEBSDLarge.yaml +++ b/em_database/datasets/NiEBSDLarge.yaml @@ -5,7 +5,7 @@ NiEBSDLarge: source: https://raw.githubusercontent.com/pyxem/kikuchipy-data/bcab8f7a4ffdb86a97f14e2327a4813d3156a85e/nickel_ebsd_large/ checksum: md5:51d6bc0f5ff23dcb0c1a8e1f4c52d4d4 file: patterns_v2.h5 - data_size: 15.4 MB + size_bytes: 15368325 detector_manufacturer: NORDIF detector: UF-1100 technique: EBSD diff --git a/em_database/datasets/PdCuSiCrystallization.yaml b/em_database/datasets/PdCuSiCrystallization.yaml index 8974253..df66352 100644 --- a/em_database/datasets/PdCuSiCrystallization.yaml +++ b/em_database/datasets/PdCuSiCrystallization.yaml @@ -12,7 +12,7 @@ PdCuSiCrystallization: source: https://zenodo.org/records/15490547/files checksum: md5:80ec7f95ec250106c586debf5d814325 file: PdCuSiCrystalization-zip.zspy - data_size: 5.75 GB + size_bytes: 5748299565 detector_manufacturer: Direct Electron detector: CeleritasXS voltage: 200 kV diff --git a/em_database/datasets/PdNiP.yaml b/em_database/datasets/PdNiP.yaml index 945e77f..0f69b48 100644 --- a/em_database/datasets/PdNiP.yaml +++ b/em_database/datasets/PdNiP.yaml @@ -4,7 +4,7 @@ PdNiPGlass: source: https://zenodo.org/records/15490547/files checksum: md5:b16375d23eda3da60ea2ff36cc11c5bd file: PdNiP.zspy - data_size: 304.6 MB + size_bytes: 304627953 detector_manufacturer: Direct Electron detector: DE-16 microscope_vendor: Thermo Fisher Scientific diff --git a/em_database/datasets/SPEDAg.yaml b/em_database/datasets/SPEDAg.yaml index 2633b99..5068e17 100644 --- a/em_database/datasets/SPEDAg.yaml +++ b/em_database/datasets/SPEDAg.yaml @@ -4,7 +4,7 @@ SPEDAg: source: https://zenodo.org/records/15490547/files checksum: md5:8556346543fc19f0ef9bdc0f4a6619b5 file: SPED-Ag.zspy - data_size: 214.0 MB + size_bytes: 213985149 detector_manufacturer: Quantum Detectors detector: Merlin microscope_vendor: JEOL diff --git a/em_database/datasets/ZrNbPercipitate.yaml b/em_database/datasets/ZrNbPercipitate.yaml index 16bb57e..433dc7d 100644 --- a/em_database/datasets/ZrNbPercipitate.yaml +++ b/em_database/datasets/ZrNbPercipitate.yaml @@ -4,7 +4,7 @@ ZrNbPrecipitate: source: https://zenodo.org/records/15490547/files checksum: md5:7012a2bdf564f4fb25299af05412723f file: ZrNbPercipitate.zspy - data_size: 394.2 MB + size_bytes: 394213247 detector_manufacturer: Direct Electron detector: DE-16 microscope_vendor: Thermo Fisher Scientific diff --git a/em_database/datasets/json-schema.json b/em_database/datasets/json-schema.json index 8a21011..4dc5234 100644 --- a/em_database/datasets/json-schema.json +++ b/em_database/datasets/json-schema.json @@ -14,42 +14,25 @@ "type": "string", "format": "uri" }, + "file": { + "type": "string" + }, "checksum": { "type": "string", "pattern": "^md5:[A-Fa-f0-9]{32}$" }, - "file": { - "type": "string" - }, - "data_size": { - "type": "string" + "size_bytes": { + "type": "integer", + "minimum": 0 }, "detector_manufacturer": { - "type": "string", - "options": { - "enum": [ - "Gatan", - "Thermo Fisher Scientific", - "Direct Electron", - "Dectris", - "Quantum Detectors", - "TVIPS", - "Other" - ] - } + "type": "string" }, "detector": { "type": "string" }, "microscope_vendor": { - "type": "string", - "enum": [ - "Thermo Fisher Scientific", - "JEOL", - "Hitachi", - "Zeiss", - "Other" - ] + "type": "string" }, "microscope_model": { "type": "string" @@ -57,9 +40,6 @@ "camera_length": { "type": "string" }, - "folder": { - "type": "string" - }, "voltage": { "type": "string", "pattern": "^[0-9]+\\s?kV$" diff --git a/em_database/datasets/vendors.yaml b/em_database/datasets/vendors.yaml new file mode 100644 index 0000000..160efb9 --- /dev/null +++ b/em_database/datasets/vendors.yaml @@ -0,0 +1,14 @@ +microscope_vendor: + - Hitachi + - JEOL + - Thermo Fisher Scientific + - Zeiss +detector_manufacturer: + - Allied Vision + - Dectris + - Direct Electron + - Gatan + - NORDIF + - Quantum Detectors + - Thermo Fisher Scientific + - TVIPS diff --git a/em_database/downloadable_dataset.py b/em_database/downloadable_dataset.py index ad7cfbe..852f424 100644 --- a/em_database/downloadable_dataset.py +++ b/em_database/downloadable_dataset.py @@ -1,17 +1,38 @@ +import atexit +import os +import sys import threading +import warnings from concurrent.futures import Future, ThreadPoolExecutor from pathlib import Path -from typing import Optional, Union +from typing import Any, ClassVar, Protocol import pooch -import os + +from em_database.metadata import DatasetMetadata + + +class Progress(Protocol): + """What pooch drives while streaming a file, and what the widgets provide.""" + + @property + def total(self) -> int: ... + + @total.setter + def total(self, value: int) -> None: ... + + def update(self, n: int) -> None: ... + + def reset(self) -> None: ... + + def close(self) -> None: ... # A single, lazily-created thread pool shared by every dataset. Background # downloads run here so that a notebook cell returns immediately instead of # blocking on the network. It is created on first use so that simply importing # the package costs nothing. -_executor: Optional[ThreadPoolExecutor] = None +_executor: ThreadPoolExecutor | None = None _executor_lock = threading.Lock() @@ -25,9 +46,28 @@ def _get_executor() -> ThreadPoolExecutor: max_workers=4, thread_name_prefix="em_database_download", ) + atexit.register(_shutdown_executor) return _executor +def _shutdown_executor() -> None: + """Drop queued downloads on the way out of the interpreter. + + The pool's threads are non-daemon, so Python joins them at exit: queue up + several large datasets and the process hangs on the last one long after you + asked it to stop, with Ctrl-C going to the main thread and being ignored. + Cancelling clears everything not yet started. + + A transfer already streaming still runs to completion - there is no way to + interrupt pooch mid-read from here - so this shortens the wait rather than + removing it. The widget's toast has a cancel button for that case. + """ + global _executor + executor, _executor = _executor, None + if executor is not None: + executor.shutdown(wait=False, cancel_futures=True) + + # Subclass the *concrete* Path for this OS (WindowsPath / PosixPath) so that # instances are real ``pathlib.Path`` objects. This matters: consumers such as # ``hyperspy.load`` do ``isinstance(arg, Path)`` and reject anything else, so a @@ -35,9 +75,26 @@ def _get_executor() -> ThreadPoolExecutor: # the download path — but a Path subclass does. _ConcretePath = type(Path()) +# Downloads in flight, keyed by the destination path. Keeping this off the +# instance is what makes a derived path behave: pathlib builds a brand new +# object for ``handle.parent / handle.name``, and an instance attribute would +# not survive that, so the copy would report itself finished and never wait. +_PENDING: dict[str, "Future[Path]"] = {} +_PENDING_LOCK = threading.Lock() + + +def _pending_key(path: object) -> str: + return os.path.normcase(os.path.abspath(str(path))) + -class DownloadFuture(_ConcretePath): - """The path to a dataset that is downloading on a background thread. +def _release_pending(key: str, future: "Future[Path]") -> None: + with _PENDING_LOCK: + if _PENDING.get(key) is future: + del _PENDING[key] + + +class DatasetPath(_ConcretePath): + """The local path to a dataset, which may still be downloading. It is a genuine :class:`pathlib.Path` pointing at the file's final location (known before the download starts), so it can be passed anywhere a path is @@ -46,72 +103,162 @@ class DownloadFuture(_ConcretePath): every file reader ultimately goes through) it blocks until the download has finished, re-raising any error that occurred. - For explicit control it also behaves like a future: ``done()`` checks status - without blocking, ``result()``/``wait()`` block until the file is ready. + ``done`` reports status without blocking; ``result()``/``wait()`` block + until the file is ready. A path to a file that is already on disk is simply + one that is already done, so :meth:`DownloadableDataset.download` returns + this type whether or not it downloaded anything. + + Any path pointing at the same file waits, however it was built. ``str()`` + and ``Path()`` are the exceptions: they hand back a plain value with no + download attached, so ``hs.load(str(handle))`` will not block. """ - def _attach(self, future: "Future[str]") -> "DownloadFuture": - self._future = future + @property + def _future(self) -> "Future[Path] | None": + return _PENDING.get(_pending_key(self)) + + def _attach(self, future: "Future[Path]") -> "DatasetPath": + key = _pending_key(self) + with _PENDING_LOCK: + _PENDING[key] = future + future.add_done_callback(lambda finished: _release_pending(key, finished)) return self def __fspath__(self) -> str: # is_file()/exists()/stat()/open() and every os.fspath() consumer route # through here, so blocking here makes all of them wait for the bytes. - future = getattr(self, "_future", None) + future = self._future if future is not None: future.result() # blocks; re-raises a failed download return str(self) - def result(self, timeout: Optional[float] = None) -> str: + def result(self, timeout: float | None = None) -> Path: """Block until the download finishes and return the file path.""" - future = getattr(self, "_future", None) + future = self._future if future is not None: future.result(timeout) - return str(self) + return Path(str(self)) + @property def done(self) -> bool: - """Return True if the download has finished (without blocking).""" - future = getattr(self, "_future", None) + """Whether the download has finished. Never blocks.""" + future = self._future return future.done() if future is not None else True - def wait(self, timeout: Optional[float] = None) -> "DownloadFuture": + def wait(self, timeout: float | None = None) -> "DatasetPath": """Block until the download finishes and return self (for chaining).""" self.result(timeout) return self def __repr__(self) -> str: # never block just to display the object - state = "done" if self.done() else "downloading" - return f"" + state = "done" if self.done else "downloading" + return f"" + + +class _TqdmProgress: + """A tqdm bar for a pooch download, created only once bytes start arriving. + + Two reasons not to let pooch build its own. It passes ``ncols=79``, meaning + 79 *terminal columns*, but tqdm's notebook backend reads ``ncols`` as a + pixel width and sets ``layout.width = "79px"`` - a bar squashed to 79 pixels + with its own horizontal scrollbar. And ``pooch.retrieve`` skips the + downloader entirely for a file that is already cached, so building the bar + up front would flash an empty one on every cached call; pooch assigns + ``total`` exactly once, before streaming, which is the moment there is + something worth showing. + """ + + def __init__(self, desc: str = "") -> None: + self._desc = desc + self._bar: Any = None # a tqdm.auto bar; which backend depends on the host + self._total = 0 + + @property + def total(self) -> int: + return self._total + + @total.setter + def total(self, value: int) -> None: + from tqdm.auto import tqdm + + self._total = int(value or 0) + if self._bar is None: + self._bar = tqdm( + total=self._total, + desc=self._desc, + unit="B", + unit_scale=True, + # Windows terminals do not always have the box-drawing glyphs. + ascii=sys.platform == "win32", + leave=True, + ) + else: + self._bar.reset(total=self._total) + + def update(self, n: int) -> None: + if self._bar is not None: + self._bar.update(n) + + def reset(self) -> None: + if self._bar is not None: + self._bar.reset(total=self._total) + + def close(self) -> None: + if self._bar is not None: + self._bar.close() + self._bar = None class DownloadableDataset: + """A downloadable dataset, described by the YAML entry in :attr:`_spec`. + + The generated subclasses in :mod:`em_database.data` carry their entry as + ``_spec``; keyword arguments override it for a single instance. - def __init__(self, - source: str, - file: str, - checksum:str=None, - license:str=None, - quality:str=None, - data_size:str=None, - doi:str=None, - description:str=None, - detector:Optional[str]=None, - detector_manufacturer:Optional[str]=None, - **kwargs): - self.source = source - self.file = file - self.checksum = checksum - self.license = license - self.quality = quality - self.doi = doi - self.data_size = data_size - self.description = description - self.metadata = kwargs - self.detector_manufacturer = detector_manufacturer - self.detector = detector + Everything the YAML declares is on :attr:`metadata` + (``ds.metadata.technique``). The four fields the download machinery itself + needs are also reachable directly, as :attr:`source`, :attr:`file`, + :attr:`checksum` and :attr:`size_bytes`. + """ + + _spec: ClassVar[dict[str, Any]] = {} + _origin: ClassVar[Path | None] = None + # Built when the class is, so a malformed YAML fails the import that reads + # it rather than making the dataset quietly disappear from the catalogue. + _metadata: ClassVar[DatasetMetadata | None] = None + + def __init__(self, **overrides: Any): + if self._metadata is not None and not overrides: + self.metadata = self._metadata + else: + self.metadata = DatasetMetadata.from_spec({**self._spec, **overrides}, self._origin) + + @property + def source(self) -> str: + return self.metadata.source + + @property + def file(self) -> str: + return self.metadata.file + + @property + def checksum(self) -> str | None: + return self.metadata.checksum + + @property + def size_bytes(self) -> int | None: + return self.metadata.size_bytes + + @property + def size(self) -> str: + """:attr:`size_bytes` formatted for display, or ``""`` if unknown.""" + return self.metadata.size def __repr__(self): - return f"<{self.__class__} url={self.source}/{self.file} bytes={self.data_size}>" + # __class__ rather than its __name__ nested one set of angle brackets + # inside another, which a list of results made unreadable. + headline = " · ".join(p for p in (self.file, self.metadata.technique, self.size) if p) + return f"<{type(self).__name__} {headline}>" def _repr_mimebundle_(self, **kwargs): """Rich display in Jupyter: an interactive card with download/metadata. @@ -120,25 +267,35 @@ def _repr_mimebundle_(self, **kwargs): """ try: from em_database.widget import card + widget = card(self) - except Exception: + except ImportError: + return {"text/plain": repr(self)} # anywidget not installed + except Exception as error: + # Still fall back - a repr that raises is worse than a plain one - + # but say so, rather than making a widget bug look like a missing + # optional dependency. + warnings.warn(f"falling back to the plain repr: {error}", stacklevel=2) return {"text/plain": repr(self)} return widget._repr_mimebundle_(**kwargs) @staticmethod - def _resolve_destination(destination: str | None) -> str: + def _resolve_destination(destination: str | os.PathLike | None) -> Path: """Return the directory the dataset should live in.""" if destination is None: from em_database import config + return config.data_dir() - return destination + return Path(destination) - def download(self, - destination: str | None = None, - progressbar:bool = True, - chunk_size:int =4096, - background:bool = True) -> Union[str, DownloadFuture]: - """ Download the dataset to the specified destination if not already present. + def download( + self, + destination: str | os.PathLike | None = None, + progressbar: bool | Progress = True, + chunk_size: int = 4096, + background: bool = True, + ) -> DatasetPath: + """Download the dataset to the specified destination if not already present. By default, this will download to the defined emdata.data_dir directory. You can set a custom default download directory with emdata.data_dir = 'your/path/here' which will @@ -149,7 +306,7 @@ def download(self, Parameters ---------- - destination : str, optional + destination : str or Path, optional The directory to download the dataset to. If None, uses the default emdata.data_dir directory, by default None. progressbar : bool, optional @@ -160,61 +317,63 @@ def download(self, background : bool, optional If True (the default), the download runs on a background thread and this returns immediately so a Jupyter cell stays responsive. The returned - :class:`DownloadFuture` is a real path pointing at the file's final + :class:`DatasetPath` is a real path pointing at the file's final location, so you can hand it straight to a loader (``hs.load(dataset.download())``): it blocks only at the point the file - is actually opened. Use ``.done()`` to poll and ``.result()`` to wait - explicitly. If False, the download blocks and returns the file path as a - plain string, exactly as before. + is actually opened. Use ``.done`` to poll and ``.result()`` to wait + explicitly. If False, the download blocks until the file is there. Returns ------- - str or DownloadFuture - When ``background`` is False, the local path to the downloaded file as a - string. When ``background`` is True (the default), a :class:`DownloadFuture` - path handle for the same location that resolves once the download finishes. + DatasetPath + The local path to the file, as a :class:`pathlib.Path` subclass that also + reports download state. With ``background`` False it is already done. """ if not background: - return self._retrieve(destination, progressbar, chunk_size) + return DatasetPath(self._retrieve(destination, progressbar, chunk_size)) # Resolve where the file will end up: an existing shared/user copy if it # is already present, otherwise the user's download location. if destination is not None: - target = os.path.join(destination, self.file) + target = Path(destination) / self.file else: - target = self.filepath() or os.path.join(self._resolve_destination(None), self.file) + target = self.filepath() or self._resolve_destination(None) / self.file # In Jupyter (with the widget installed) a background download pops a # cancelable toast; the toast's monitor replaces the plain progress bar. monitor = finish = None if progressbar: try: from em_database.widget import _attach_toast + monitor, finish = _attach_toast(type(self).__name__) except Exception: monitor = finish = None progress = monitor if monitor is not None else progressbar - future = _get_executor().submit( - self._retrieve, destination, progress, chunk_size - ) + future = _get_executor().submit(self._retrieve, destination, progress, chunk_size) if finish is not None: future.add_done_callback(finish) - return DownloadFuture(target)._attach(future) - - def _retrieve(self, - destination: str | None = None, - progressbar: bool = True, - chunk_size: int = 4096) -> str: + return DatasetPath(target)._attach(future) + + def _retrieve( + self, + destination: str | os.PathLike | None = None, + progressbar: bool | Progress = True, + chunk_size: int = 4096, + ) -> Path: """Fetch the file and return its local path (blocking). With no explicit destination, an existing system-wide/shared copy is used as-is (never re-downloaded); otherwise pooch downloads into the user's data directory. """ - if progressbar: + if progressbar is True: try: import tqdm # noqa: F401 except ImportError: print("`tqdm` is not installed, progress bar will be disabled.") progressbar = False + else: + # Our own bar rather than pooch's; see _TqdmProgress. + progressbar = _TqdmProgress(self.file) if destination is None: shared = self._find_shared() if shared is not None: @@ -224,45 +383,64 @@ def _retrieve(self, destination = self._resolve_destination(destination) # Instantiate an Http downloader with a custom user agent headers = {"User-Agent": "em_database (https://github.com/CSSFrancis/em_data)"} - downloader = pooch.HTTPDownloader(progressbar=progressbar, - chunk_size=chunk_size, - headers= headers) - filepath = pooch.retrieve( - url=self.source +"/"+ self.file, - known_hash=self.checksum, - fname=self.file, - path=destination, - downloader=downloader + downloader = pooch.HTTPDownloader( + progressbar=progressbar, # pyright: ignore[reportArgumentType] + chunk_size=chunk_size, + headers=headers, ) - return filepath - - def _find_shared(self) -> str | None: + try: + filepath = pooch.retrieve( + url=self.source + "/" + self.file, + known_hash=self.checksum, + fname=self.file, + path=destination, + downloader=downloader, # pyright: ignore[reportArgumentType] + ) + finally: + # pooch only closes the bar on the happy path, so a failed or + # cancelled download would leave it hanging open. + if isinstance(progressbar, _TqdmProgress): + progressbar.close() + return Path(filepath) + + def _find_shared(self) -> Path | None: """Path to an existing copy in a shared/system data dir, or None.""" from em_database import config + for directory in config.shared_data_dirs(): - candidate = os.path.join(directory, self.file) - if os.path.exists(candidate): + candidate = directory / self.file + if candidate.exists(): return candidate return None - def filepath(self) -> str: - """ Return the local file path of the dataset if present. + def filepaths(self) -> list[Path]: + """Every copy of the dataset on disk, in search order. - Looks in the shared/system data locations first, then the user's data - directory. Returns None if the dataset is not downloaded anywhere. """ + A dataset can be in more than one place at once - a shared install and + your own download of the same file - and which one gets used is only a + matter of the search order. :meth:`filepath` returns the winner; this + returns all of them, so a caller can tell the difference between the + one copy that is shared and a shared copy that you also have your own + of. + """ from em_database import config - for directory in config.data_search_dirs(): - candidate = os.path.join(directory, self.file) - if os.path.exists(candidate): - return candidate - return None + + return [d / self.file for d in config.data_search_dirs() if (d / self.file).exists()] + + def filepath(self) -> Path | None: + """Return the local file path of the dataset if present. + + Looks in the shared/system data locations first, then the user's data + directory. Returns None if the dataset is not downloaded anywhere.""" + found = self.filepaths() + return found[0] if found else None def delete(self, destination: str | None = None) -> bool: - """ Delete the downloaded file if it is present. + """Delete the downloaded file if it is present. Parameters ---------- - destination : str, optional + destination : str or Path, optional The directory the dataset was downloaded to. If None, uses the default emdata.data_dir directory, by default None. @@ -271,8 +449,8 @@ def delete(self, destination: str | None = None) -> bool: bool True if a file was removed, False if there was nothing to delete. """ - path = os.path.join(self._resolve_destination(destination), self.file) - if os.path.exists(path): - os.remove(path) + path = self._resolve_destination(destination) / self.file + if path.exists(): + path.unlink() return True return False diff --git a/em_database/metadata.py b/em_database/metadata.py new file mode 100644 index 0000000..bde038c --- /dev/null +++ b/em_database/metadata.py @@ -0,0 +1,244 @@ +"""The dataset record, typed. + +``datasets/json-schema.json`` closes the dataset entry with +``additionalProperties: false``, so an entry is a fixed set of fields rather +than an open bag. :class:`DatasetMetadata` is that set, and +:meth:`DatasetMetadata.from_spec` is the only way one is built: it turns a +parsed YAML mapping into the record and refuses anything the schema would. + +This module also owns the small amount of shared knowledge about where the +dataset files live - :func:`dataset_files`, :func:`load_schema`, +:func:`load_vendors` - so the loader, the stub generator, the docs form and the +tests all read the same directory the same way. +""" + +from __future__ import annotations + +import difflib +import textwrap +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field, fields +from pathlib import Path +from typing import Any + +import yaml + +DATASETS_DIR = Path(__file__).parent / "datasets" +SCHEMA_PATH = DATASETS_DIR / "json-schema.json" +VENDORS_PATH = DATASETS_DIR / "vendors.yaml" + +# Files in datasets/ that are not dataset collections. +NON_DATASET_FILES = frozenset({VENDORS_PATH.name}) + +REQUIRED_FIELDS = ("description", "source", "file") + +# Wrap width for __str__; narrow enough to stay readable in a notebook cell. +_STR_WIDTH = 88 + + +def dataset_files() -> list[Path]: + """Every dataset collection YAML, sorted by name.""" + return sorted(p for p in DATASETS_DIR.rglob("*.y*ml") if p.name not in NON_DATASET_FILES) + + +def load_schema() -> dict[str, Any]: + """The JSON schema the dataset YAML is validated against.""" + import json + + return json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + + +def load_vendors() -> dict[str, list[str]]: + """The canonical vendor and detector-manufacturer names. + + Suggestions, not a closed list - see :func:`check_vendor`. + """ + return yaml.safe_load(VENDORS_PATH.read_text(encoding="utf-8")) + + +def check_vendor(value: str, known: Iterable[str], cutoff: float = 0.8) -> tuple[str, str] | None: + """``(level, message)`` for a vendor string, or ``None`` if it is known. + + A name close to a known one is a spelling mistake and is an ``"error"``; + one that is nothing like any of them is a genuinely new vendor and is a + ``"warning"`` asking for it to be added to ``vendors.yaml``. A hard list + would fail the new vendor and block a contributor who has done nothing + wrong, so only the case that is always a mistake fails. + + Similarity is the check's weak point on short names - ``"JOEL"`` for + ``"JEOL"`` scores too low to be called a typo and comes back as a warning. + """ + known = list(known) + if not value or value in known: + return None + folded = {_fold(k): k for k in known} + if _fold(value) in folded: + return ("error", f"{value!r} should be spelled {folded[_fold(value)]!r}") + close = difflib.get_close_matches(value, known, n=1, cutoff=cutoff) + if close: + return ("error", f"{value!r} looks like a misspelling of {close[0]!r}") + return ("warning", f"{value!r} is not in vendors.yaml; add it there if it is a new vendor") + + +def _fold(value: str) -> str: + return "".join(value.split()).casefold() + + +_SIZE_UNITS = ("B", "kB", "MB", "GB", "TB", "PB") + + +def format_size(size_bytes: int | None) -> str: + """A byte count as SI units for display: ``1104287335`` -> ``"1.10 GB"``.""" + if size_bytes is None: + return "" + value = float(size_bytes) + unit = _SIZE_UNITS[0] + for unit in _SIZE_UNITS: + if value < 1000 or unit == _SIZE_UNITS[-1]: + break + value /= 1000 + if unit == "B": + return f"{int(value)} B" + # Enough decimals for three significant figures, so a size stays readable + # whether it is 42.0 kB or 1.10 GB. + return f"{value:.{max(1, 3 - len(str(int(value))))}f} {unit}" + + +def _where(origin: Path | str | None) -> str: + return str(origin) if origin else "dataset entry" + + +@dataclass(frozen=True) +class Author: + """One author of a dataset.""" + + affiliation: str + orcid: str | None = None + + @classmethod + def from_spec( + cls, name: str, spec: Mapping[str, Any], origin: Path | str | None = None + ) -> Author: + allowed = {f.name for f in fields(cls)} + unknown = sorted(set(spec) - allowed) + if unknown: + raise TypeError( + f"{_where(origin)}: author {name!r} has unknown field(s) " + f"{', '.join(repr(k) for k in unknown)}; allowed: {', '.join(sorted(allowed))}" + ) + if "affiliation" not in spec: + raise TypeError(f"{_where(origin)}: author {name!r} is missing 'affiliation'") + return cls(affiliation=str(spec["affiliation"]), orcid=spec.get("orcid")) + + +@dataclass(frozen=True, repr=False) +class DatasetMetadata: + """Everything a dataset YAML entry declares. + + Field order matches ``json-schema.json``; a test asserts the two stay in + step. + + ``repr=False`` because the generated one runs to about a thousand + characters - nearly all of it the description - which is no use as the + output of a bare ``ds.metadata`` in a notebook. :meth:`__repr__` identifies + the record in one line; ``print(ds.metadata)`` gives the readable form. + """ + + description: str + source: str + file: str + checksum: str | None = None + size_bytes: int | None = None + detector_manufacturer: str | None = None + detector: str | None = None + microscope_vendor: str | None = None + microscope_model: str | None = None + camera_length: str | None = None + voltage: str | None = None + license: str | None = None + technique: str | None = None + doi: str | None = None + tags: tuple[str, ...] = () + authors: Mapping[str, Author] = field(default_factory=dict) + + @classmethod + def from_spec( + cls, spec: Mapping[str, Any], origin: Path | str | None = None + ) -> DatasetMetadata: + """Build a record from a parsed YAML entry. + + ``origin`` is the file the entry came from; it only appears in error + messages, where it is the difference between a useful complaint and a + ``TypeError`` from somewhere inside an import. + """ + allowed = {f.name for f in fields(cls)} + unknown = sorted(set(spec) - allowed) + if unknown: + raise TypeError( + f"{_where(origin)}: unknown field(s) {', '.join(repr(k) for k in unknown)}; " + f"allowed: {', '.join(sorted(allowed))}" + ) + missing = [name for name in REQUIRED_FIELDS if spec.get(name) is None] + if missing: + raise TypeError( + f"{_where(origin)}: missing required field(s) " + f"{', '.join(repr(k) for k in missing)}" + ) + values = dict(spec) + size_bytes = values.get("size_bytes") + values["size_bytes"] = None if size_bytes is None else int(size_bytes) + values["tags"] = tuple(str(t) for t in values.get("tags") or ()) + values["authors"] = { + str(name): Author.from_spec(str(name), entry or {}, origin) + for name, entry in (values.get("authors") or {}).items() + } + return cls(**values) + + @property + def size(self) -> str: + """:attr:`size_bytes` formatted for display, or ``""`` if unknown.""" + return format_size(self.size_bytes) + + def __repr__(self) -> str: + """One line naming the file, what it is and how big: enough to tell two + records apart in a list without printing a paragraph of description.""" + headline = " · ".join(p for p in (self.file, self.technique, self.size) if p) + return f"<{type(self).__name__} {headline}>" + + def __str__(self) -> str: + """The whole record, wrapped, with the empty fields left out.""" + headline = " · ".join(p for p in (self.file, self.technique, self.size) if p) + head = [headline] + if self.description: + head.append(textwrap.fill(self.description, width=_STR_WIDTH)) + + rows: list[tuple[str, str]] = [] + for entry in fields(self): + if entry.name in ("description", "file", "technique", "size_bytes"): + continue # in the headline already, or shown as size_bytes below + value = getattr(self, entry.name) + if not value: + continue + if entry.name == "authors": + value = "; ".join(f"{n} ({a.affiliation})" for n, a in value.items()) + elif entry.name == "tags": + value = ", ".join(value) + rows.append((entry.name, str(value))) + if self.size_bytes is not None: + rows.append(("size_bytes", f"{self.size_bytes} ({self.size})")) + + label_width = max((len(name) for name, _ in rows), default=0) + body = [ + textwrap.fill( + value, + width=_STR_WIDTH, + initial_indent=f"{name:>{label_width}}: ", + subsequent_indent=" " * (label_width + 2), + # A URL or a checksum is one long token: overrun the width + # rather than split it, so it can still be copied. + break_long_words=False, + break_on_hyphens=False, + ) + for name, value in rows + ] + return "\n\n".join(part for part in ("\n".join(head), "\n".join(body)) if part) diff --git a/em_database/query.py b/em_database/query.py new file mode 100644 index 0000000..95aaa44 --- /dev/null +++ b/em_database/query.py @@ -0,0 +1,130 @@ +"""Querying the catalogue from Python. + +The browser widget has always had a search box; this is that search, callable, +returning dataset objects rather than the dicts the widget draws. + +The module is ``query`` and not ``search`` so that it cannot collide with the +:func:`search` function it exports: a submodule and a function of the same name +both live in the ``em_database`` namespace, and importing the submodule wins. + +:func:`search` matches the same lowercased blob that +:func:`em_database.catalogue.entry` builds for the widget, by the same rule - +every whitespace-separated term has to appear somewhere - so a query typed into +the widget and the same query passed here select the same datasets. The widget +keeps matching in the browser rather than calling this, because a round trip to +the kernel on every keystroke would make typing lag; what is shared is the blob +and the rule, which is where the two would otherwise drift apart. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from em_database import catalogue +from em_database.downloadable_dataset import DownloadableDataset + +#: Fields :func:`filter` accepts. The first group is the dataset's own metadata; +#: ``downloaded`` and ``location`` describe what is on this machine instead. +FILTER_FIELDS = ( + "technique", + "detector", + "detector_manufacturer", + "microscope_vendor", + "microscope_model", + "camera_length", + "voltage", + "license", + "doi", + "tags", + "authors", + "downloaded", + "location", +) + + +def list_datasets() -> list[DownloadableDataset]: + """Every dataset in the index, sorted by name. + + Not ``datasets()``: ``em_database/datasets/`` is a subpackage, and the + import machinery overwrites a same-named attribute on the parent the moment + anything imports it - which Sphinx does, walking every submodule. + + >>> import em_database + >>> len(em_database.list_datasets()) > 0 + True + """ + return [ds for _, ds in catalogue.datasets()] + + +def search(query: str) -> list[DownloadableDataset]: + """Datasets whose metadata contains every word of ``query``. + + The text searched is everything the widget searches: name, description, + technique, detector, microscope, voltage, licence, DOI, file name, tags, + authors and their affiliations. Matching is case-insensitive, and terms may + match different fields - ``"jeol eels"`` finds EELS datasets taken on a + JEOL. An empty query returns everything. + """ + terms = query.lower().split() + if not terms: + return list_datasets() + found = [] + for name, ds in catalogue.datasets(): + blob = catalogue.entry(name, ds)["search"] + if all(term in blob for term in terms): + found.append(ds) + return found + + +def filter(**criteria: Any) -> list[DownloadableDataset]: # noqa: A001 + """Datasets matching every one of ``criteria``. + + Fields are those in :data:`FILTER_FIELDS`. String comparisons are exact but + case-insensitive; ``tags`` and ``authors`` test membership; passing a list + matches any of its values:: + + em_database.filter(technique="4D-STEM") + em_database.filter(technique="4D-STEM", tags="Strain") + em_database.filter(microscope_vendor=["JEOL", "Hitachi"]) + em_database.filter(downloaded=True) + em_database.filter(location="shared") + + Several criteria are combined with and. An unknown field raises rather than + being ignored, so a typo cannot quietly return the whole index. + + The name shadows the builtin ``filter`` inside this module, which is why it + is reached as ``em_database.filter``; nothing here needs the builtin. + """ + unknown = sorted(set(criteria) - set(FILTER_FIELDS)) + if unknown: + raise TypeError( + f"unknown filter field(s) {', '.join(repr(k) for k in unknown)}; " + f"choose from {', '.join(FILTER_FIELDS)}" + ) + found = [] + for name, ds in catalogue.datasets(): + row = catalogue.entry(name, ds) + if all(_matches(_value(ds, row, field), wanted) for field, wanted in criteria.items()): + found.append(ds) + return found + + +def _value(ds: DownloadableDataset, row: dict, field: str) -> Any: + """The value ``field`` is filtered on for one dataset.""" + if field in ("downloaded", "location"): + return row[field] + return getattr(ds.metadata, field) + + +def _matches(value: Any, wanted: Any) -> bool: + if wanted is None or isinstance(wanted, bool): + return value == wanted + wanted_values = [wanted] if isinstance(wanted, str) else list(wanted) + if isinstance(value, (tuple, list, Mapping)): + # tags and authors: a dataset matches if it carries any one of them. + have = {str(v).casefold() for v in value} + return any(str(w).casefold() in have for w in wanted_values) + if value is None: + return False + return any(str(value).casefold() == str(w).casefold() for w in wanted_values) diff --git a/em_database/static/browser.css b/em_database/static/browser.css index 3ba9959..dd4ade9 100644 --- a/em_database/static/browser.css +++ b/em_database/static/browser.css @@ -116,6 +116,7 @@ .emdb-glyph { flex: 0 0 auto; width: 12px; text-align: center; font-size: 10px; } .emdb-glyph.on { color: var(--emdb-green); } .emdb-glyph.off { color: var(--emdb-muted); } +.emdb-glyph.shared { color: var(--emdb-blue); } .emdb-name { font-weight: 600; white-space: nowrap; } .emdb-meta { flex: 1; color: var(--emdb-subtext); font-size: 11px; text-align: right; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } @@ -137,6 +138,7 @@ .emdb-d-status { margin: 10px 0 12px; display: flex; align-items: center; gap: 10px; } .emdb-d-badge { font-size: 11.5px; color: var(--emdb-subtext); } .emdb-d-badge.on { color: var(--emdb-green); font-weight: 600; } +.emdb-d-badge.shared { color: var(--emdb-blue); font-weight: 600; } .emdb-delete { cursor: pointer; font-family: var(--emdb-font); font-size: 11px; font-weight: 600; color: var(--emdb-red); background: transparent; diff --git a/em_database/static/browser.js b/em_database/static/browser.js index b0be3d7..c895e06 100644 --- a/em_database/static/browser.js +++ b/em_database/static/browser.js @@ -126,6 +126,8 @@ function render({ model, el: root }) { // `item.search` is a lowercased blob of every field (name, description, // detector, microscope, tags, authors + affiliations, license, …), so a // query like "Carter Francis" matches on author, not just the name. + // em_database.search() matches this same blob by this same rule; matching + // stays here rather than in the kernel so typing never waits on a round trip. const blob = item.search || item.name.toLowerCase(); return state.search.toLowerCase().split(/\s+/).every((term) => blob.includes(term)); } @@ -204,8 +206,9 @@ function render({ model, el: root }) { const isActive = active.has(item.name); const row = el("div", "emdb-row" + (state.selected === item.name ? " selected" : "")); const meta = [item.size, item.shape].filter(Boolean).join(" · "); - row.appendChild(el("span", "emdb-glyph " + (item.downloaded ? "on" : "off"), - item.downloaded ? "●" : "○")); + const glyph = el("span", "emdb-glyph " + glyphClass(item), item.downloaded ? "●" : "○"); + if (item.location === "shared") glyph.title = sharedTitle(item); + row.appendChild(glyph); row.appendChild(el("span", "emdb-name", esc(item.name))); row.appendChild(el("span", "emdb-meta", esc(meta))); row.appendChild(drawAction(item, isActive)); @@ -215,6 +218,18 @@ function render({ model, el: root }) { return row; } + // A shared copy and your own can both exist; the tooltip names each. + function sharedTitle(item) { + const lines = ["installed system-wide: " + item.path]; + if (item.user_path) lines.push("your copy: " + item.user_path); + return lines.join("\n"); + } + + function glyphClass(item) { + if (!item.downloaded) return "off"; + return item.location === "shared" ? "shared" : "on"; + } + function drawAction(item, isActive) { const wrap = el("span", "emdb-actions"); if (item.downloaded) { @@ -251,7 +266,17 @@ function render({ model, el: root }) { // status / action line const statusRow = el("div", "emdb-d-status"); - if (item.downloaded) { + if (item.downloaded && item.location === "shared") { + const badge = el("span", "emdb-d-badge shared", item.user_path ? "● shared + yours" : "● shared"); + badge.title = sharedTitle(item); + statusRow.appendChild(badge); + if (item.user_path) { + const del = el("button", "emdb-delete", "Delete yours"); + del.title = "Remove your copy (" + item.user_path + "). The shared one stays."; + del.addEventListener("click", () => cmd("delete", { name: item.name })); + statusRow.appendChild(del); + } + } else if (item.downloaded) { statusRow.appendChild(el("span", "emdb-d-badge on", "● downloaded")); const del = el("button", "emdb-delete", "Delete"); del.title = "Remove the downloaded file from disk"; diff --git a/em_database/static/card.js b/em_database/static/card.js index d54aefa..c8e6661 100644 --- a/em_database/static/card.js +++ b/em_database/static/card.js @@ -124,6 +124,17 @@ function render({ model, el: root }) { const status = el("div", "emdb-d-status"); if (downloading) { status.appendChild(el("span", "emdb-d-badge", "downloading…")); + } else if (it.downloaded && it.location === "shared") { + const badge = el("span", "emdb-d-badge shared", it.user_path ? "● shared + yours" : "● shared"); + badge.title = "installed system-wide: " + it.path + + (it.user_path ? "\nyour copy: " + it.user_path : ""); + status.appendChild(badge); + if (it.user_path) { + const del = el("button", "emdb-delete", "Delete yours"); + del.title = "Remove your copy (" + it.user_path + "). The shared one stays."; + del.addEventListener("click", () => cmd("delete")); + status.appendChild(del); + } } else if (it.downloaded) { status.appendChild(el("span", "emdb-d-badge on", "● downloaded")); const del = el("button", "emdb-delete", "Delete"); diff --git a/em_database/tests/__init__.py b/em_database/tests/__init__.py index bdab649..e69de29 100644 --- a/em_database/tests/__init__.py +++ b/em_database/tests/__init__.py @@ -1,7 +0,0 @@ -import pytest - -from em_database import get_data_dir, set_data_dir, reset_data_dir - -def test_download_directory(): - reset_data_dir() - print(get_data_dir()) diff --git a/em_database/tests/conftest.py b/em_database/tests/conftest.py index 68e55c5..a13dde5 100644 --- a/em_database/tests/conftest.py +++ b/em_database/tests/conftest.py @@ -4,6 +4,7 @@ the legacy ``EM_DATABASE_DATA_DIR`` env var cleared, so the developer's real settings never affect a test and a test never writes to the real config. """ + import pytest diff --git a/em_database/tests/test_directory.py b/em_database/tests/test_directory.py index d9ffbdc..1684ead 100644 --- a/em_database/tests/test_directory.py +++ b/em_database/tests/test_directory.py @@ -4,7 +4,7 @@ dataset in the index rather than a large one. """ -import os +from pathlib import Path import pytest @@ -12,7 +12,7 @@ from em_database import data from em_database.tests.test_load_data import TINY_DATASET -DEFAULT_DIR = os.path.join(os.path.expanduser("~"), "em_database") +DEFAULT_DIR = Path.home() / "em_database" @pytest.fixture(autouse=True) @@ -26,9 +26,14 @@ def test_get_data_dir(): assert em_database.get_data_dir() == DEFAULT_DIR +def test_reset_data_dir_returns_to_the_default(): + em_database.reset_data_dir() + assert em_database.get_data_dir() == DEFAULT_DIR + + def test_set_data_dir(tmp_path): em_database.set_data_dir(str(tmp_path)) - assert em_database.get_data_dir() == str(tmp_path) + assert em_database.get_data_dir() == tmp_path def test_reset_data_dir(tmp_path): @@ -42,7 +47,7 @@ def test_saving_to_configured_dir(tmp_path): em_database.set_data_dir(str(tmp_path)) dataset = getattr(data, TINY_DATASET)() dest = dataset.download(progressbar=False, background=False) - assert os.path.exists(os.path.join(str(tmp_path), dataset.file)) + assert (tmp_path / dataset.file).exists() # a second download must reuse the file rather than refetch it assert dataset.download(progressbar=False, background=False) == dest @@ -53,7 +58,7 @@ def test_saving_to_explicit_dir(tmp_path): em_database.set_data_dir(str(tmp_path / "configured")) dataset = getattr(data, TINY_DATASET)() dest = dataset.download(destination=str(other), progressbar=False, background=False) - assert "elsewhere" in dest + assert "elsewhere" in str(dest) assert (other / dataset.file).exists() @@ -63,4 +68,4 @@ def test_filepath_reports_missing_and_present(tmp_path): dataset = getattr(data, TINY_DATASET)() assert dataset.filepath() is None dataset.download(progressbar=False, background=False) - assert dataset.filepath() == os.path.join(str(tmp_path), dataset.file) + assert dataset.filepath() == tmp_path / dataset.file diff --git a/em_database/tests/test_load_data.py b/em_database/tests/test_load_data.py index 723c0fd..bf09e03 100644 --- a/em_database/tests/test_load_data.py +++ b/em_database/tests/test_load_data.py @@ -19,14 +19,26 @@ import pytest import em_database.data as data +from em_database import catalogue from em_database.data import MgONanoCrystals, NiEBSDLarge -from em_database.downloadable_dataset import DownloadFuture +from em_database.downloadable_dataset import ( + _PENDING, + DatasetPath, + DownloadableDataset, + _get_executor, + _pending_key, + _shutdown_executor, + _TqdmProgress, +) try: - from quantem.core.io.file_readers import read_4dstem + from quantem.core.io.file_readers import ( # pyright: ignore[reportMissingImports] + read_4dstem, + ) QUANTEM_AVAILABLE = True except ImportError: + read_4dstem = None QUANTEM_AVAILABLE = False # The smallest file in the index (34 kB). Used wherever a test needs a real @@ -44,12 +56,16 @@ def _head(url, timeout=60): return urllib.request.urlopen(request, timeout=timeout) +@pytest.mark.network @pytest.mark.parametrize("name", ALL_DATASETS) def test_source_url_resolves(name): """Every dataset's source URL must still exist. This is what actually breaks over time - a Zenodo record superseded, a GitHub ref rewritten - and it is invisible until someone tries to download. + Deselected by default: it is 20 HEAD requests, and running it on every leg + of the push matrix meant 120 of them per push. The weekly check_sources + workflow runs it instead. """ dataset = getattr(data, name)() url = f"{dataset.source}/{dataset.file}" @@ -60,6 +76,14 @@ def test_source_url_resolves(name): except urllib.error.URLError as error: # pragma: no cover - transient pytest.skip(f"{name}: network unavailable ({error.reason})") assert response.status == 200, f"{name}: {url} returned {response.status}" + # The header the declared size came from. Checking it also catches the + # source file being replaced, which is otherwise invisible until someone's + # checksum fails. + length = response.headers.get("Content-Length") + if length is not None and dataset.size_bytes is not None: + assert int(length) == dataset.size_bytes, ( + f"{name}: {url} is {int(length)} bytes, but the YAML declares {dataset.size_bytes}" + ) @pytest.mark.parametrize("name", ALL_DATASETS) @@ -68,11 +92,11 @@ def test_metadata_is_complete(name): dataset = getattr(data, name)() assert dataset.source, f"{name} has no source" assert dataset.file, f"{name} has no file" - assert dataset.description, f"{name} has no description" + assert dataset.metadata.description, f"{name} has no description" assert dataset.checksum and dataset.checksum.startswith("md5:"), ( - f"{name} has no md5 checksum, so a corrupt or truncated download " - f"would go unnoticed" + f"{name} has no md5 checksum, so a corrupt or truncated download would go unnoticed" ) + assert dataset.size_bytes, f"{name} has no size_bytes" def test_download_verifies_checksum(tmp_path): @@ -80,8 +104,10 @@ def test_download_verifies_checksum(tmp_path): dataset = getattr(data, TINY_DATASET)() path = dataset.download(destination=tmp_path, progressbar=False, background=False) assert (tmp_path / dataset.file).exists() - assert isinstance(path, str) - assert path == str(tmp_path / dataset.file) + assert isinstance(path, DatasetPath) + assert isinstance(path, Path) + assert path.done is True # nothing pending, so the handle needs no waiting + assert path == tmp_path / dataset.file def test_download_default_returns_path_handle(tmp_path): @@ -89,13 +115,13 @@ def test_download_default_returns_path_handle(tmp_path): that is a real ``Path`` and resolves to the downloaded file.""" dataset = getattr(data, TINY_DATASET)() handle = dataset.download(destination=tmp_path, progressbar=False) - assert isinstance(handle, DownloadFuture) + assert isinstance(handle, DatasetPath) assert isinstance(handle, Path) # Using it as a path blocks until the bytes are there, then behaves normally. assert os.fspath(handle) == str(tmp_path / dataset.file) assert handle.is_file() assert (tmp_path / dataset.file).exists() - assert handle.done() + assert handle.done def test_download_handle_is_nonblocking_then_blocks_on_use(tmp_path, monkeypatch): @@ -113,18 +139,89 @@ def slow_retrieve(destination=None, progressbar=True, chunk_size=4096): monkeypatch.setattr(dataset, "_retrieve", slow_retrieve) handle = dataset.download(destination=tmp_path, progressbar=False) - assert started.wait(2) # the worker thread really started - assert handle.done() is False # returned without waiting for it + assert started.wait(2) # the worker thread really started + assert handle.done is False # returned without waiting for it assert not (tmp_path / dataset.file).exists() # Consuming the path blocks until the worker finishes, then resolves. assert Path(os.fspath(handle)).read_bytes() == b"payload" - assert handle.done() is True + assert handle.done is True + + +def test_download_handle_derived_paths_also_wait(tmp_path, monkeypatch): + """A path rebuilt from the handle names the same file, so it must wait too.""" + dataset = getattr(data, TINY_DATASET)() + started = threading.Event() + + def slow_retrieve(destination=None, progressbar=True, chunk_size=4096): + started.set() + time.sleep(0.4) + target = tmp_path / dataset.file + target.write_bytes(b"payload") + return str(target) + + monkeypatch.setattr(dataset, "_retrieve", slow_retrieve) + handle = dataset.download(destination=tmp_path, progressbar=False) + assert started.wait(2) + + derived = handle.parent / handle.name + assert derived is not handle + assert derived.done is False + assert Path(os.fspath(derived)).read_bytes() == b"payload" + + +def test_a_path_that_is_not_downloading_never_waits(tmp_path, monkeypatch): + """Only the file being fetched is pending - its directory is not.""" + dataset = getattr(data, TINY_DATASET)() + + def slow_retrieve(destination=None, progressbar=True, chunk_size=4096): + time.sleep(0.3) + target = tmp_path / dataset.file + target.write_bytes(b"payload") + return str(target) + + monkeypatch.setattr(dataset, "_retrieve", slow_retrieve) + handle = dataset.download(destination=tmp_path, progressbar=False) + assert handle.parent.done is True + handle.wait() + + +def test_finished_downloads_leave_no_pending_entry(tmp_path): + dataset = getattr(data, TINY_DATASET)() + handle = dataset.download(destination=tmp_path, progressbar=False) + handle.wait() + key = _pending_key(handle) + # the done-callback that clears the entry runs just after result() returns + for _ in range(200): + if key not in _PENDING: + break + time.sleep(0.01) + assert key not in _PENDING + + +def test_generated_class_can_be_subclassed(): + base = getattr(data, TINY_DATASET) + + class Subclass(base): + pass + + assert Subclass().file == base().file + + +def test_keyword_overrides_leave_the_class_spec_alone(): + base = getattr(data, TINY_DATASET) + overridden = base(checksum="md5:" + "0" * 32) + assert overridden.checksum == "md5:" + "0" * 32 + assert base().checksum != overridden.checksum + + +def test_a_dataset_without_a_source_is_an_error(): + with pytest.raises(TypeError): + DownloadableDataset() def test_download_handle_propagates_errors(tmp_path): """A failed background download raises when the handle is consumed.""" - dataset = getattr(data, TINY_DATASET)() - dataset.checksum = "md5:" + "0" * 32 + dataset = getattr(data, TINY_DATASET)(checksum="md5:" + "0" * 32) handle = dataset.download(destination=tmp_path, progressbar=False) with pytest.raises(Exception): os.fspath(handle) @@ -142,8 +239,7 @@ def test_download_is_cached(tmp_path): def test_download_rejects_a_bad_checksum(tmp_path): """A wrong checksum must raise rather than hand back the file.""" - dataset = getattr(data, TINY_DATASET)() - dataset.checksum = "md5:" + "0" * 32 + dataset = getattr(data, TINY_DATASET)(checksum="md5:" + "0" * 32) with pytest.raises(Exception): dataset.download(destination=tmp_path, progressbar=False, background=False) @@ -165,6 +261,97 @@ def test_download_mgo_nanocrystals(tmp_path): @pytest.mark.slow @pytest.mark.skipif(not QUANTEM_AVAILABLE, reason="quantem is not installed") def test_quantem_loading(tmp_path): + assert read_4dstem is not None dataset = MgONanoCrystals() file_path = dataset.download(destination=tmp_path, progressbar=False, background=False) read_4dstem(file_path) + + +def test_tqdm_bar_waits_for_pooch_to_set_a_total(): + """pooch skips the downloader for a cached file, so the bar must be lazy.""" + progress = _TqdmProgress("demo") + assert progress._bar is None # nothing shown yet + progress.total = 100 + assert progress._bar is not None + progress.update(50) + assert progress._bar.n == 50 + progress.reset() + progress.update(100) + assert progress._bar.n == 100 + progress.close() + assert progress._bar is None + + +def test_tqdm_bar_is_not_squashed_to_79_pixels(monkeypatch): + """pooch passes ncols=79 meaning terminal columns, but tqdm's notebook + backend reads ncols as a pixel width - a bar 79 pixels wide, which is where + the horizontal scrollbar in Jupyter comes from. Ours passes no ncols.""" + notebook_tqdm = pytest.importorskip("tqdm.notebook") + + squashed = notebook_tqdm.tqdm(total=1000, ncols=79) + assert squashed.container.layout.width == "79px" # what pooch produces + squashed.close() + + # _TqdmProgress builds through tqdm.auto; force the notebook backend so the + # widget layout is the thing under test rather than the terminal one. + monkeypatch.setattr("tqdm.auto.tqdm", notebook_tqdm.tqdm) + progress = _TqdmProgress("demo") + progress.total = 1000 + assert progress._bar.container.layout.width is None # ipywidgets default + progress.close() + + +def test_progressbar_true_is_swapped_for_our_own_bar(tmp_path, monkeypatch): + """`progressbar=True` must reach pooch as a Progress object, not as True.""" + seen = {} + + def fake_retrieve(**kwargs): + seen["downloader"] = kwargs["downloader"] + target = tmp_path / kwargs["fname"] + target.write_bytes(b"payload") + return str(target) + + monkeypatch.setattr("pooch.retrieve", fake_retrieve) + dataset = getattr(data, TINY_DATASET)() + dataset._retrieve(destination=tmp_path, progressbar=True) + + assert isinstance(seen["downloader"].progressbar, _TqdmProgress) + assert seen["downloader"].progressbar is not True + + +def test_shutdown_cancels_queued_downloads(): + """The pool's threads are non-daemon, so anything still queued at exit would + hold the interpreter open. Queued work is cancelled; started work is not.""" + executor = _get_executor() + started = threading.Event() + release = threading.Event() + + def block(): + started.set() + release.wait(timeout=5) + + running = [executor.submit(block) for _ in range(4)] # fill every worker + queued = [executor.submit(block) for _ in range(8)] + started.wait(timeout=5) + try: + _shutdown_executor() + assert any(f.cancelled() for f in queued) + assert not any(f.cancelled() for f in running) + finally: + release.set() + + +def test_a_malformed_entry_warns_instead_of_vanishing(monkeypatch): + """catalogue.datasets() used to swallow every exception, so a bad dataset + just disappeared from the browser with nothing said.""" + broken = type( + "BrokenDataset", + (DownloadableDataset,), + {"_spec": {"description": "d", "source": "s"}, "_metadata": None}, # no 'file' + ) + monkeypatch.setattr(data, "BrokenDataset", broken, raising=False) + monkeypatch.setattr(data, "__all__", [*data.__all__, "BrokenDataset"]) + + with pytest.warns(UserWarning, match="BrokenDataset"): + found = dict(catalogue.datasets()) + assert "BrokenDataset" not in found # skipped, but not silently diff --git a/em_database/tests/test_metadata.py b/em_database/tests/test_metadata.py new file mode 100644 index 0000000..6934e04 --- /dev/null +++ b/em_database/tests/test_metadata.py @@ -0,0 +1,224 @@ +"""Tests for the dataset record and the files it is built from. + +The shipped YAML is the product this package exists to hand out, and until now +nothing checked it against the schema sitting next to it. These tests are that +check, plus the ones that keep the schema, the dataclass and the vendor list +from drifting apart - three descriptions of the same record, in three files. +""" + +import dataclasses + +import pytest +import yaml + +from em_database.metadata import ( + Author, + DatasetMetadata, + check_vendor, + dataset_files, + format_size, + load_schema, + load_vendors, +) + +jsonschema = pytest.importorskip("jsonschema") + +DATASET_FILES = dataset_files() +SCHEMA = load_schema() +ENTRY_SCHEMA = SCHEMA["patternProperties"]["^.+$"] +VENDORS = load_vendors() + + +def entries(): + """``(file, name, spec)`` for every entry in every dataset YAML.""" + for path in DATASET_FILES: + document = yaml.safe_load(path.read_text(encoding="utf-8")) + for name, spec in document.items(): + yield path, name, spec + + +ENTRIES = list(entries()) + + +def test_datasets_are_found(): + assert DATASET_FILES, "no dataset YAML found" + assert all(p.name != "vendors.yaml" for p in DATASET_FILES) + + +@pytest.mark.parametrize("path", DATASET_FILES, ids=lambda p: p.name) +def test_yaml_matches_schema(path): + jsonschema.validate(yaml.safe_load(path.read_text(encoding="utf-8")), SCHEMA) + + +@pytest.mark.parametrize("name", [name for _, name, _ in ENTRIES]) +def test_entry_builds_a_record(name): + path, _, spec = next(e for e in ENTRIES if e[1] == name) + metadata = DatasetMetadata.from_spec(spec, path) + assert metadata.description and metadata.source and metadata.file + + +def test_schema_and_dataclass_agree(): + """The schema and :class:`DatasetMetadata` describe one record, in two files.""" + assert list(ENTRY_SCHEMA["properties"]) == [ + f.name for f in dataclasses.fields(DatasetMetadata) + ] + assert set(ENTRY_SCHEMA["required"]) == { + f.name + for f in dataclasses.fields(DatasetMetadata) + if f.default is dataclasses.MISSING and f.default_factory is dataclasses.MISSING + } + + +def test_author_schema_and_dataclass_agree(): + author_schema = ENTRY_SCHEMA["properties"]["authors"]["patternProperties"]["^.+$"] + assert list(author_schema["properties"]) == [f.name for f in dataclasses.fields(Author)] + + +def test_unknown_field_names_the_file_and_the_key(): + with pytest.raises(TypeError, match=r"somewhere\.yaml.*'folder'"): + DatasetMetadata.from_spec( + {"description": "d", "source": "s", "file": "f", "folder": "x"}, + "somewhere.yaml", + ) + + +def test_missing_required_field_is_an_error(): + with pytest.raises(TypeError, match="'file'"): + DatasetMetadata.from_spec({"description": "d", "source": "s"}) + + +def test_author_without_affiliation_is_an_error(): + with pytest.raises(TypeError, match="affiliation"): + DatasetMetadata.from_spec( + {"description": "d", "source": "s", "file": "f", "authors": {"Jane Doe": {}}} + ) + + +def test_tags_and_authors_are_converted(): + metadata = DatasetMetadata.from_spec( + { + "description": "d", + "source": "s", + "file": "f", + "tags": ["One", "Two"], + "authors": {"Jane Doe": {"affiliation": "Somewhere", "orcid": None}}, + } + ) + assert metadata.tags == ("One", "Two") + assert metadata.authors["Jane Doe"] == Author(affiliation="Somewhere") + + +@pytest.mark.parametrize( + ("size_bytes", "expected"), + [ + (None, ""), + (0, "0 B"), + (999, "999 B"), + (1000, "1.00 kB"), + (34043, "34.0 kB"), + (466089, "466.1 kB"), + (104291721, "104.3 MB"), + (1104287335, "1.10 GB"), + (5748299565, "5.75 GB"), + ], +) +def test_format_size(size_bytes, expected): + assert format_size(size_bytes) == expected + + +@pytest.mark.parametrize(("path", "name", "spec"), ENTRIES, ids=[e[1] for e in ENTRIES]) +def test_declared_vendors_are_spelled_correctly(path, name, spec): + """A vendor close to a known one is a typo; one nothing like it is just new.""" + for field, known in ( + ("detector_manufacturer", VENDORS["detector_manufacturer"]), + ("microscope_vendor", VENDORS["microscope_vendor"]), + ): + result = check_vendor(spec.get(field, ""), known) + if result is None: + continue + level, message = result + assert level != "error", f"{path.name}: {name}: {field}: {message}" + + +def test_check_vendor_tells_a_typo_from_a_new_vendor(): + known = ["Direct Electron", "Gatan"] + + def level(value): + result = check_vendor(value, known) + return result[0] if result else None + + assert level("Gatan") is None + assert level("") is None + assert level("Direct electron") == "error" + assert level("Direct Electon") == "error" + assert level("Nion") == "warning" + + +def test_vendors_yaml_covers_what_the_datasets_declare(): + for field in ("detector_manufacturer", "microscope_vendor"): + declared = {spec[field] for _, _, spec in ENTRIES if spec.get(field)} + assert declared <= set(VENDORS[field]) + + +def _record(**overrides): + spec = { + "description": "A description.", + "source": "https://example.com/files", + "file": "d.zspy", + } + return DatasetMetadata.from_spec({**spec, **overrides}) + + +def test_repr_is_one_short_identifying_line(): + """The generated dataclass repr is ~1000 chars of mostly description, which + is useless as the output of a bare `ds.metadata` in a notebook.""" + metadata = _record(technique="4D-STEM", size_bytes=12492298) + text = repr(metadata) + assert "\n" not in text + assert len(text) < 100 + assert text == "" + assert "A description." not in text + + +def test_repr_drops_the_parts_it_does_not_have(): + assert repr(_record()) == "" + + +def test_str_shows_the_record_and_omits_empty_fields(): + metadata = _record(technique="EELS", license="CC-BY-4.0", tags=["One", "Two"]) + text = str(metadata) + assert "A description." in text + assert "license: CC-BY-4.0" in text + assert "tags: One, Two" in text + assert "checksum" not in text # not set, so not shown + assert "camera_length" not in text + + +def test_str_shows_bytes_and_the_readable_size_together(): + assert "size_bytes: 12492298 (12.5 MB)" in str(_record(size_bytes=12492298)) + + +def test_str_formats_authors_with_affiliations(): + metadata = _record(authors={"Jane Doe": {"affiliation": "Somewhere"}}) + assert "authors: Jane Doe (Somewhere)" in str(metadata) + + +def test_str_never_breaks_a_url_across_lines(): + """A wrapped URL cannot be copied, so long tokens overrun the width instead.""" + url = "https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets" + assert url in str(_record(source=url)) + + +def test_str_wraps_a_long_value_under_its_label(): + tags = [f"Tag Number {n}" for n in range(12)] + lines = str(_record(tags=tags)).splitlines() + tag_lines = [line for line in lines if "Tag Number" in line] + assert len(tag_lines) > 1 # wrapped + assert all(len(line) <= 88 for line in tag_lines) + assert tag_lines[1].startswith(" " * 8) # continuation is indented under the label + + +def test_str_without_a_description(): + text = str(DatasetMetadata(description="", source="https://example.com", file="d.zspy")) + assert text.startswith("d.zspy\n\n") + assert "source: https://example.com" in text diff --git a/em_database/tests/test_query.py b/em_database/tests/test_query.py new file mode 100644 index 0000000..3198316 --- /dev/null +++ b/em_database/tests/test_query.py @@ -0,0 +1,165 @@ +"""Tests for the Python query API. + +The conftest fixture isolates the settings and clears the shared/data env vars, +so ``downloaded`` / ``location`` here describe a tmp dir rather than whatever the +developer happens to have downloaded. +""" + +import types + +import pytest + +import em_database +from em_database import catalogue +from em_database.data import BilayerWS2 +from em_database.downloadable_dataset import DownloadableDataset +from em_database.query import FILTER_FIELDS + + +def names(found): + return sorted(type(ds).__name__ for ds in found) + + +def test_datasets_returns_objects_not_rows(): + found = em_database.list_datasets() + assert found + assert all(isinstance(ds, DownloadableDataset) for ds in found) + assert names(found) == sorted(n for n, _ in catalogue.datasets()) + + +def test_search_matches_every_term_across_fields(): + """The widget's rule: all terms must appear, but not in one field.""" + found = names(em_database.search("jeol eels")) + assert found + for name in found: + dataset = catalogue.resolve(name) + assert dataset is not None + blob = catalogue.entry(name, dataset)["search"] + assert "jeol" in blob and "eels" in blob + + +def test_search_is_case_insensitive(): + assert names(em_database.search("AMORPHOUS")) == names(em_database.search("amorphous")) + + +def test_search_finds_a_dataset_by_author(): + """Proof the blob reaches past the name - authors are not in the class name.""" + ds = BilayerWS2() + author = next(iter(ds.metadata.authors)) + assert type(ds).__name__ in names(em_database.search(author)) + + +def test_search_uses_the_same_blob_and_rule_as_the_widget(): + query = "direct electron" + terms = query.lower().split() + expected = sorted( + name + for name, dataset in catalogue.datasets() + if all(t in catalogue.entry(name, dataset)["search"] for t in terms) + ) + assert names(em_database.search(query)) == expected + + +def test_empty_search_returns_everything(): + assert names(em_database.search(" ")) == names(em_database.list_datasets()) + + +def test_search_with_no_hits_is_empty(): + assert em_database.search("definitelynotadataset") == [] + + +def test_filter_is_exact_but_case_insensitive(): + exact = names(em_database.filter(technique="4D-STEM")) + assert exact + assert names(em_database.filter(technique="4d-stem")) == exact + # exact, not substring: "4D" must not match "4D-STEM" + assert em_database.filter(technique="4D") == [] + + +def test_filter_combines_criteria_with_and(): + both = names(em_database.filter(technique="4D-STEM", tags="Strain")) + technique_only = names(em_database.filter(technique="4D-STEM")) + assert both + assert set(both) < set(technique_only) + + +def test_filter_accepts_a_list_as_any_of(): + jeol = set(names(em_database.filter(microscope_vendor="JEOL"))) + tfs = set(names(em_database.filter(microscope_vendor="Thermo Fisher Scientific"))) + either = set(names(em_database.filter(microscope_vendor=["JEOL", "Thermo Fisher Scientific"]))) + assert either == jeol | tfs + assert jeol and tfs + + +def test_filter_on_tags_tests_membership(): + for ds in em_database.filter(tags="Strain"): + assert "Strain" in ds.metadata.tags + + +def test_filter_on_authors_tests_membership(): + ds = BilayerWS2() + author = next(iter(ds.metadata.authors)) + assert type(ds).__name__ in names(em_database.filter(authors=author)) + + +def test_filter_rejects_an_unknown_field(): + with pytest.raises(TypeError, match="techniqu"): + em_database.filter(techniqu="4D-STEM") + + +def test_filter_fields_are_all_actually_filterable(): + """Every advertised field has to work, not just be listed.""" + for field in FILTER_FIELDS: + em_database.filter(**{field: None if field == "location" else "nothing-matches-this"}) + + +def test_filter_on_downloaded_and_location(tmp_path, monkeypatch): + shared, user = tmp_path / "shared", tmp_path / "user" + shared.mkdir() + user.mkdir() + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared)) + em_database.set_data_dir(str(user), persist=False) + + assert em_database.filter(downloaded=True) == [] + assert names(em_database.filter(downloaded=False)) == names(em_database.list_datasets()) + + ds = BilayerWS2() + (user / ds.file).write_bytes(b"mine") + assert names(em_database.filter(downloaded=True)) == [type(ds).__name__] + assert names(em_database.filter(location="user")) == [type(ds).__name__] + assert em_database.filter(location="shared") == [] + + (shared / ds.file).write_bytes(b"theirs") + assert names(em_database.filter(location="shared")) == [type(ds).__name__] + assert em_database.filter(location="user") == [] + + +def test_the_query_api_is_on_the_top_level_namespace(): + for name in ("list_datasets", "search", "filter"): + assert name in em_database.__all__ + assert callable(getattr(em_database, name)) + + +def test_public_names_survive_a_submodule_walk(): + """A submodule and a top-level function cannot share a name. + + Importing ``em_database.`` makes the import machinery bind that + submodule onto the parent package, replacing any function of the same name. + Sphinx's autosummary walks every submodule, so this is not hypothetical - it + is what took the docs build down when the function was called ``datasets()`` + next to the ``em_database/datasets/`` subpackage. + """ + import importlib + import pkgutil + + for found in pkgutil.walk_packages(em_database.__path__, "em_database."): + try: + importlib.import_module(found.name) + except ImportError: + continue # an optional dependency, not a name clash + + for name in em_database.__all__: + exported = getattr(em_database, name) + assert not isinstance(exported, types.ModuleType) or name in ("data", "settings"), ( + f"em_database.{name} was replaced by a submodule of the same name" + ) diff --git a/em_database/tests/test_settings.py b/em_database/tests/test_settings.py index 36b1806..be1e22f 100644 --- a/em_database/tests/test_settings.py +++ b/em_database/tests/test_settings.py @@ -3,34 +3,37 @@ The autouse fixture in conftest.py isolates each test to an empty settings file and clears the legacy env var, so these exercise the mechanism in isolation. """ + +from pathlib import Path + import em_database from em_database import config def test_defaults_when_nothing_configured(): assert em_database.get_data_dir() == config._default_data_dir() - assert em_database.settings["data_dir"] == config._default_data_dir() + assert em_database.settings["data_dir"] == str(config._default_data_dir()) def test_live_object_is_immediate_like_rcparams(tmp_path): em_database.settings["data_dir"] = str(tmp_path / "live") - assert em_database.get_data_dir() == str(tmp_path / "live") # no save needed - assert not config.config_path().exists() # not persisted + assert em_database.get_data_dir() == tmp_path / "live" # no save needed + assert not config.config_path().exists() # not persisted def test_set_data_dir_persists_across_sessions(tmp_path): target = str(tmp_path / "data") - em_database.set_data_dir(target) # persist=True by default - assert em_database.get_data_dir() == target + em_database.set_data_dir(target) # persist=True by default + assert em_database.get_data_dir() == Path(target) assert config._read_file()["data_dir"] == target - config.settings.reload() # a fresh "session" - assert em_database.get_data_dir() == target + config.settings.reload() # a fresh "session" + assert em_database.get_data_dir() == Path(target) def test_set_data_dir_session_only_is_not_persisted(tmp_path): target = str(tmp_path / "x") em_database.set_data_dir(target, persist=False) - assert em_database.get_data_dir() == target + assert em_database.get_data_dir() == Path(target) assert not config.config_path().exists() config.settings.reload() assert em_database.get_data_dir() == config._default_data_dir() # forgotten @@ -54,13 +57,13 @@ def test_generic_setting_roundtrips(): def test_saving_other_settings_keeps_the_default_dynamic(): """Persisting an unrelated setting must NOT freeze the default data_dir into the file - otherwise the default would stop being obeyed.""" - em_database.set_setting("quality", "high") # triggers a save() + em_database.set_setting("quality", "high") # triggers a save() stored = config._read_file() - assert stored == {"quality": "high"} # data_dir default not written + assert stored == {"quality": "high"} # data_dir default not written assert em_database.get_data_dir() == config._default_data_dir() def test_legacy_env_var_still_seeds(tmp_path, monkeypatch): monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path / "fromenv")) config.settings.reload() # a fresh "import" with the env var set - assert em_database.get_data_dir() == str(tmp_path / "fromenv") + assert em_database.get_data_dir() == tmp_path / "fromenv" diff --git a/em_database/tests/test_shared_data.py b/em_database/tests/test_shared_data.py index 7beaf71..2cf5afc 100644 --- a/em_database/tests/test_shared_data.py +++ b/em_database/tests/test_shared_data.py @@ -4,15 +4,19 @@ env vars, so these exercise the resolution logic in isolation (no network - the "downloads" here find a pre-placed file). """ + import os import em_database from em_database import catalogue, config +from em_database.downloadable_dataset import DownloadableDataset from em_database.tests.test_load_data import TINY_DATASET -def _dataset(): - return catalogue.resolve(TINY_DATASET) +def _dataset() -> DownloadableDataset: + ds = catalogue.resolve(TINY_DATASET) + assert ds is not None + return ds def test_settings_live_in_dot_em_database(monkeypatch): @@ -30,11 +34,11 @@ def test_shared_dir_is_searched_before_the_user_dir(tmp_path, monkeypatch): em_database.set_data_dir(str(user), persist=False) ds = _dataset() - assert ds.filepath() is None # nowhere yet + assert ds.filepath() is None # nowhere yet (user / ds.file).write_bytes(b"user") - assert ds.filepath() == str(user / ds.file) # found in the user dir + assert ds.filepath() == user / ds.file # found in the user dir (shared / ds.file).write_bytes(b"shared") - assert ds.filepath() == str(shared / ds.file) # shared/system wins + assert ds.filepath() == shared / ds.file # shared/system wins def test_download_uses_a_shared_copy_without_refetching(tmp_path, monkeypatch): @@ -45,21 +49,77 @@ def test_download_uses_a_shared_copy_without_refetching(tmp_path, monkeypatch): em_database.set_data_dir(str(user), persist=False) ds = _dataset() - (shared / ds.file).write_bytes(b"payload") # pre-installed system-wide + (shared / ds.file).write_bytes(b"payload") # pre-installed system-wide path = ds.download(background=False) - assert path == str(shared / ds.file) # used the shared copy - assert not (user / ds.file).exists() # nothing downloaded to the user dir + assert path == shared / ds.file # used the shared copy + assert not (user / ds.file).exists() # nothing downloaded to the user dir def test_search_order_is_shared_then_user(tmp_path, monkeypatch): a, b = tmp_path / "a", tmp_path / "b" monkeypatch.setenv("EM_DATABASE_SHARED_DIR", os.pathsep.join([str(a), str(b)])) em_database.set_data_dir(str(tmp_path / "user"), persist=False) - assert config.data_search_dirs() == [str(a), str(b), str(tmp_path / "user")] + assert config.data_search_dirs() == [a, b, tmp_path / "user"] def test_system_config_file_contributes_shared_dirs(tmp_path, monkeypatch): system_file = tmp_path / "system.yaml" system_file.write_text(f"data_dir: {tmp_path / 'sitewide'}\n", encoding="utf-8") monkeypatch.setenv("EM_DATABASE_SYSTEM_CONFIG", str(system_file)) - assert str(tmp_path / "sitewide") in config.shared_data_dirs() + assert tmp_path / "sitewide" in config.shared_data_dirs() + + +def test_filepaths_reports_every_copy(tmp_path, monkeypatch): + """A shared install and your own download of the same file coexist.""" + shared, user = tmp_path / "shared", tmp_path / "user" + shared.mkdir() + user.mkdir() + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared)) + em_database.set_data_dir(str(user), persist=False) + + ds = _dataset() + assert ds.filepaths() == [] + (shared / ds.file).write_bytes(b"shared") + assert ds.filepaths() == [shared / ds.file] + (user / ds.file).write_bytes(b"user") + assert ds.filepaths() == [shared / ds.file, user / ds.file] + assert ds.filepath() == shared / ds.file # the winner is still the shared one + + +def test_entry_exposes_a_user_copy_hiding_behind_a_shared_one(tmp_path, monkeypatch): + """The widget needs to see the copy delete() would remove, not just the winner.""" + shared, user = tmp_path / "shared", tmp_path / "user" + shared.mkdir() + user.mkdir() + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared)) + em_database.set_data_dir(str(user), persist=False) + + ds = _dataset() + (shared / ds.file).write_bytes(b"shared") + row = catalogue.entry(TINY_DATASET, ds) + assert row["location"] == "shared" + assert row["user_path"] == "" # nothing of yours to delete + + (user / ds.file).write_bytes(b"user") + row = catalogue.entry(TINY_DATASET, ds) + assert row["location"] == "shared" # the shared copy is still the one in use + assert row["user_path"] == str(user / ds.file) + + assert ds.delete() is True # deletes yours + row = catalogue.entry(TINY_DATASET, ds) + assert row["user_path"] == "" + assert (shared / ds.file).exists() # the shared copy is untouched + assert row["downloaded"] is True # and still resolves + + +def test_entry_user_path_when_there_is_no_share(tmp_path, monkeypatch): + user = tmp_path / "user" + user.mkdir() + monkeypatch.delenv("EM_DATABASE_SHARED_DIR", raising=False) + em_database.set_data_dir(str(user), persist=False) + + ds = _dataset() + (user / ds.file).write_bytes(b"user") + row = catalogue.entry(TINY_DATASET, ds) + assert row["location"] == "user" + assert row["user_path"] == str(user / ds.file) diff --git a/em_database/tests/test_widget.py b/em_database/tests/test_widget.py index 489d2c6..5c7de33 100644 --- a/em_database/tests/test_widget.py +++ b/em_database/tests/test_widget.py @@ -4,6 +4,7 @@ and read declared metadata). The widget tests need ``anywidget``; the one real download is marked ``slow``. """ + import threading import pytest @@ -18,6 +19,7 @@ # catalogue # --------------------------------------------------------------------------- + def test_catalogue_groups_and_orders_by_technique(): cat = catalogue.catalogue() assert cat["n_total"] > 0 @@ -33,18 +35,48 @@ def test_catalogue_groups_and_orders_by_technique(): def test_catalogue_entry_has_expected_fields(): ds = catalogue.resolve(TINY_DATASET) + assert ds is not None row = catalogue.entry(TINY_DATASET, ds) - for key in ("name", "technique", "size", "downloaded", "path", "description", - "detector", "microscope", "voltage", "tags", "source", "file"): + for key in ( + "name", + "technique", + "size", + "downloaded", + "path", + "description", + "detector", + "microscope", + "voltage", + "tags", + "source", + "file", + ): assert key in row assert row["name"] == TINY_DATASET assert row["technique"] == "STEM" assert isinstance(row["downloaded"], bool) +def test_catalogue_entry_reports_where_the_file_came_from(tmp_path, monkeypatch): + shared, user = tmp_path / "shared", tmp_path / "user" + shared.mkdir() + user.mkdir() + monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared)) + em_database.set_data_dir(str(user), persist=False) + + ds = catalogue.resolve(TINY_DATASET) + assert ds is not None + assert catalogue.entry(TINY_DATASET, ds)["location"] is None + (user / ds.file).write_bytes(b"x") + assert catalogue.entry(TINY_DATASET, ds)["location"] == "user" + (shared / ds.file).write_bytes(b"x") + assert catalogue.entry(TINY_DATASET, ds)["location"] == "shared" + + def test_catalogue_downloaded_flag_tracks_the_file(tmp_path): em_database.set_data_dir(str(tmp_path), persist=False) ds = catalogue.resolve(TINY_DATASET) + assert ds is not None assert catalogue.entry(TINY_DATASET, ds)["downloaded"] is False (tmp_path / ds.file).write_bytes(b"x") # pretend it is downloaded assert catalogue.entry(TINY_DATASET, ds)["downloaded"] is True @@ -54,6 +86,7 @@ def test_catalogue_downloaded_flag_tracks_the_file(tmp_path): # widget # --------------------------------------------------------------------------- + def _browser(): pytest.importorskip("anywidget") return em_database.browse() @@ -106,24 +139,32 @@ def test_command_update_routes_through_real_comm_handler(monkeypatch): widget = _browser() got = [] monkeypatch.setattr(widget, "_start_download", lambda name: got.append(name)) - msg = {"content": {"data": {"method": "update", - "state": {"_command": {"action": "download", "name": "X", "nonce": 1}}}}, - "buffers": []} + msg = { + "content": { + "data": { + "method": "update", + "state": {"_command": {"action": "download", "name": "X", "nonce": 1}}, + } + }, + "buffers": [], + } widget._handle_msg(msg) # the real ipywidgets handler assert got == ["X"] def test_search_blob_includes_authors_and_affiliation(): ds = catalogue.resolve("BilayerWS2") + assert ds is not None row = catalogue.entry("BilayerWS2", ds) - assert "nick hagopian" in row["search"] # author name - assert "wisconsin" in row["search"] # author affiliation - assert "4d-stem" in row["search"] # technique + assert "nick hagopian" in row["search"] # author name + assert "wisconsin" in row["search"] # author affiliation + assert "4d-stem" in row["search"] # technique def test_delete_removes_downloaded_file(tmp_path): em_database.set_data_dir(str(tmp_path), persist=False) ds = catalogue.resolve(TINY_DATASET) + assert ds is not None (tmp_path / ds.file).write_bytes(b"x") assert ds.filepath() is not None assert ds.delete() is True @@ -160,6 +201,7 @@ def test_dataset_card_is_populated_and_routes(monkeypatch): from em_database.widget import card ds = catalogue.resolve(TINY_DATASET) + assert ds is not None widget = card(ds) assert widget.info["name"] == TINY_DATASET assert widget.info["technique"] == "STEM" @@ -172,8 +214,10 @@ def test_dataset_card_is_populated_and_routes(monkeypatch): def test_dataset_display_is_a_widget_card(): pytest.importorskip("anywidget") ds = catalogue.resolve(TINY_DATASET) + assert ds is not None bundle = ds._repr_mimebundle_() mimes = bundle[0] if isinstance(bundle, tuple) else bundle + assert mimes is not None assert "application/vnd.jupyter.widget-view+json" in mimes @@ -185,7 +229,9 @@ def _boom(_dataset): monkeypatch.setattr(widget_mod, "card", _boom) ds = catalogue.resolve(TINY_DATASET) + assert ds is not None bundle = ds._repr_mimebundle_() + assert bundle is not None assert "text/plain" in bundle @@ -195,16 +241,16 @@ def test_settings_widget_edits_and_persists(tmp_path): from em_database.widget import settings_widget widget = settings_widget() - assert widget.data_dir == em_database.get_data_dir() + assert widget.data_dir == str(em_database.get_data_dir()) widget._command = {"action": "save", "data_dir": str(tmp_path / "d"), "nonce": 1} - assert em_database.get_data_dir() == str(tmp_path / "d") - assert config._read_file()["data_dir"] == str(tmp_path / "d") # persisted + assert em_database.get_data_dir() == tmp_path / "d" + assert config._read_file()["data_dir"] == str(tmp_path / "d") # persisted assert widget.data_dir == str(tmp_path / "d") widget._command = {"action": "session", "data_dir": str(tmp_path / "s"), "nonce": 2} - assert em_database.get_data_dir() == str(tmp_path / "s") - assert config._read_file()["data_dir"] == str(tmp_path / "d") # NOT persisted + assert em_database.get_data_dir() == tmp_path / "s" + assert config._read_file()["data_dir"] == str(tmp_path / "d") # NOT persisted widget._command = {"action": "reset", "nonce": 3} assert em_database.get_data_dir() == config._default_data_dir() @@ -214,6 +260,7 @@ def test_settings_display_is_a_widget(): pytest.importorskip("anywidget") bundle = em_database.settings._repr_mimebundle_() mimes = bundle[0] if isinstance(bundle, tuple) else bundle + assert mimes is not None assert "application/vnd.jupyter.widget-view+json" in mimes @@ -221,15 +268,17 @@ def test_notebook_detection_and_colab_enable_are_safe(): """The frontend helpers must be no-ops off a notebook (e.g. under pytest), so nothing breaks when em_database is imported in plain Python.""" import em_database.widget as widget_mod + assert widget_mod._in_notebook() is False widget_mod._enable_colab_widgets() # must not raise when not on Colab - widget_mod._prepare_frontend() # idempotent, safe + widget_mod._prepare_frontend() # idempotent, safe def test_attach_toast_is_noop_outside_jupyter(): """Outside a Jupyter kernel there is no toast, so a bare download is unaffected.""" import em_database.widget as widget_mod + monitor, finish = widget_mod._attach_toast("Foo") assert monitor is None and finish is None @@ -246,17 +295,19 @@ def test_download_toasts_plumbing(): monitor, token = toasts.begin("Foo") assert toasts.downloads[token] == {"label": "Foo", "done": 0, "total": 0} - ok = Future(); ok.set_result("path") - toasts.finish(token, ok) # success -> toast cleared + ok = Future() + ok.set_result("path") + toasts.finish(token, ok) # success -> toast cleared assert token not in toasts.downloads - monitor2, token2 = toasts.begin("Bar") # cancel sets the event + monitor2, token2 = toasts.begin("Bar") # cancel sets the event toasts._command = {"action": "cancel", "token": token2, "nonce": 1} assert monitor2._cancel.is_set() - bad = Future(); bad.set_exception(RuntimeError("boom")) + bad = Future() + bad.set_exception(RuntimeError("boom")) _, token3 = toasts.begin("Baz") - toasts.finish(token3, bad) # failure -> error toast + toasts.finish(token3, bad) # failure -> error toast assert toasts.downloads[token3]["error"] == "boom" @@ -268,4 +319,5 @@ def test_widget_download_end_to_end(tmp_path): assert future is not None future.result(timeout=120) # block until the background download finishes ds = catalogue.resolve(TINY_DATASET) + assert ds is not None assert (tmp_path / ds.file).exists() diff --git a/em_database/utils.py b/em_database/utils.py deleted file mode 100644 index e69de29..0000000 diff --git a/em_database/widget.py b/em_database/widget.py index 6672cb3..9c04868 100644 --- a/em_database/widget.py +++ b/em_database/widget.py @@ -10,11 +10,13 @@ is not installed. Importing this module never imports anywidget at module load, so ``import em_database`` stays cheap and dependency-light. """ + from __future__ import annotations import itertools import threading import time +import warnings from pathlib import Path from em_database import catalogue as _catalogue @@ -41,7 +43,9 @@ def _quiet_pooch(): return try: import logging + import pooch + pooch.get_logger().setLevel(logging.WARNING) _pooch_quieted = True except Exception: @@ -60,7 +64,8 @@ def _enable_colab_widgets(): if _colab_enabled: return try: - from google.colab import output # importable only on Colab + from google.colab import output # pyright: ignore[reportMissingImports] + output.enable_custom_widget_manager() except Exception: pass @@ -204,8 +209,8 @@ def _delete(self, name): if ds is not None: try: ds.delete() - except Exception: - pass + except OSError as error: # read-only dir, permissions, a vanished file + warnings.warn(f"could not delete {name}: {error}", stacklevel=2) self.refresh() # -- downloads ------------------------------------------------------ @@ -222,12 +227,8 @@ def _start_download(self, name): # and it also covers the cached case where no bytes ever flow. self._set_progress(token, name, 0, 0) monitor = _WidgetProgress(self, token, name, cancel) - future = _get_executor().submit( - ds.download, progressbar=monitor, background=False - ) - future.add_done_callback( - lambda f, tk=token, nm=name: self._finish_download(tk, nm, f) - ) + future = _get_executor().submit(ds.download, progressbar=monitor, background=False) + future.add_done_callback(lambda f, tk=token, nm=name: self._finish_download(tk, nm, f)) return future def _finish_download(self, token, name, future): @@ -281,7 +282,7 @@ class DatasetCard(anywidget.AnyWidget): _esm = _STATIC / "card.js" _css = _STATIC / "browser.css" - info = traitlets.Dict().tag(sync=True) # the catalogue entry() dict + info = traitlets.Dict().tag(sync=True) # the catalogue entry() dict download = traitlets.Dict().tag(sync=True) # {label, done, total} | {} | {error} _command = traitlets.Dict().tag(sync=True) @@ -397,6 +398,7 @@ def browse(**kwargs): # Global toasts: a bare ``ds.download()`` in Jupyter pops a cancelable toast # --------------------------------------------------------------------------- + def _make_settings_class(): """Build the ``SettingsWidget`` class, importing anywidget lazily.""" import anywidget @@ -423,10 +425,10 @@ def __init__(self, **kwargs): self.observe(self._on_command, names="_command") def _refresh(self, status=""): - self.data_dir = config.data_dir() - self.default_dir = config._default_data_dir() + self.data_dir = str(config.data_dir()) + self.default_dir = str(config._default_data_dir()) self.config_path = str(config.config_path()) - self.search_dirs = list(config.data_search_dirs()) + self.search_dirs = [str(d) for d in config.data_search_dirs()] self.status = status def _on_command(self, change): @@ -552,7 +554,8 @@ def _in_notebook(): """True in a notebook frontend that can render widgets (Jupyter, Colab, VS Code, ...), False in plain Python or a terminal IPython.""" try: - from IPython import get_ipython + from IPython.core.getipython import get_ipython + ip = get_ipython() if ip is None: return False @@ -583,6 +586,7 @@ def _get_toasts(): if _toasts is None: _toasts = _toasts_class() from IPython.display import display + display(_toasts) except Exception: return None diff --git a/examples/changing_download_location.py b/examples/changing_download_location.py index 17b539f..4313d94 100644 --- a/examples/changing_download_location.py +++ b/examples/changing_download_location.py @@ -24,7 +24,7 @@ # Change it and remember the choice across sessions (writes # ``~/.em_database/settings.yaml``). ``set_data_dir`` persists by default; the # equivalent low-level form is ``em_database.settings[...] = ...; save()``. -em_database.set_data_dir("/big/disk/em_data") # set + persist +em_database.set_data_dir("/big/disk/em_data") # set + persist # em_database.settings["data_dir"] = "/big/disk/em_data" # the same thing # em_database.settings.save() print("Persisted directory:", em_database.get_data_dir()) diff --git a/examples/loading_hyperspy.py b/examples/loading_hyperspy.py index b6e3b26..4f81a75 100644 --- a/examples/loading_hyperspy.py +++ b/examples/loading_hyperspy.py @@ -6,6 +6,7 @@ """ import hyperspy.api as hs + from em_database.data import BilayerWS2 # Load a dataset using HyperSpy diff --git a/examples/loading_quantem.py b/examples/loading_quantem.py index 72067a6..8bb9baa 100644 --- a/examples/loading_quantem.py +++ b/examples/loading_quantem.py @@ -8,6 +8,7 @@ from quantem.core.io.file_readers import read_4dstem from em_database.data import BilayerWS2 + # Ensure the dataset is downloaded dataset = BilayerWS2() file_path = dataset.download() @@ -16,4 +17,4 @@ print(data) # %% # Display the data using quantem's built-in visualization -data.show() \ No newline at end of file +data.show() diff --git a/examples/searching_the_index.py b/examples/searching_the_index.py new file mode 100644 index 0000000..048a496 --- /dev/null +++ b/examples/searching_the_index.py @@ -0,0 +1,65 @@ +""" +Finding a Dataset +================= + +``em_database`` ships an index of every dataset it knows about, and three +functions for querying it: :func:`~em_database.datasets`, +:func:`~em_database.search` and :func:`~em_database.filter`. All three return +dataset objects, so anything they hand back can be downloaded directly. + +Nothing here touches the network - the index is metadata that ships with the +package, and querying it never opens a file. +""" + +import em_database + +# %% +# Every dataset in the index. +print(len(em_database.list_datasets()), "datasets") + +# %% +# ``search`` is the browser widget's search box, callable. It matches a +# lowercased blob of every field - name, description, technique, detector, +# microscope, tags, authors and their affiliations - and every whitespace +# separated term has to appear somewhere, though not in the same field. +for ds in em_database.search("amorphous"): + print(f"{type(ds).__name__:24s} {ds.metadata.technique:8s} {ds.size:>9s}") + +# %% +# Because terms may land in different fields, a query can combine what an +# instrument is with what the technique was. +for ds in em_database.search("jeol eels"): + print(type(ds).__name__) + +# %% +# ``filter`` matches named fields instead. Strings compare exactly but +# case-insensitively, ``tags`` and ``authors`` test membership, and a list +# means any of its values. +for ds in em_database.filter(technique="4D-STEM", tags="Strain"): + print(ds, "·", ", ".join(ds.metadata.tags)) + +# %% +for ds in em_database.filter(microscope_vendor=["JEOL", "Hitachi"]): + print(ds) + +# %% +# ``downloaded`` and ``location`` describe this machine rather than the +# dataset, which makes "what do I already have" a one-liner. +for ds in em_database.filter(downloaded=True): + print(f"{type(ds).__name__:24s} {ds.filepath()}") + +# %% +# An unknown field raises rather than being ignored - a typo that quietly +# returned the whole index would be worse than an error. +try: + em_database.filter(techniqu="4D-STEM") +except TypeError as error: + print(error) + +# %% +# The result is a dataset, so it is ready to use. Sizes are integers +# (``size_bytes``), so they sort; ``size`` is the same number for reading. +smallest = min(em_database.filter(technique="EELS"), key=lambda ds: ds.size_bytes or 0) +print(smallest) +print() +print(smallest.metadata) diff --git a/pyproject.toml b/pyproject.toml index 66cddb9..4a56adc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,17 +38,41 @@ Issues = "https://github.com/CSSFrancis/em_data/issues" [project.optional-dependencies] widget = ["anywidget>=0.9"] -tests = ["pytest", "pytest-cov", "hyperspy", "quantem", "anywidget>=0.9"] +tests = ["pytest", "pytest-cov", "hyperspy", "quantem", "anywidget>=0.9", "jsonschema"] doc = ["sphinx", "sphinx-rtd-theme","sphinx-gallery", "quantem", "hyperspy","pydata_sphinx_theme","sphinx_design",] +dev = [ + "ruff", + "basedpyright", + "pre-commit", + "pytest", + "jsonschema", + "anywidget>=0.9", +] + + +[tool.ruff] +line-length = 99 +# data/__init__.pyi is generated by em_database._create_stubs; CI checks it is +# current, which only works if nothing else rewrites it. +extend-exclude = ["*.md", "*.pyi"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "I"] +[tool.basedpyright] +typeCheckingMode = "standard" +include = ["em_database"] +exclude = ["**/__pycache__", "doc", "build"] +pythonVersion = "3.12" [tool.pytest.ini_options] # `slow` tests pull down the large datasets. They are deselected by default so # CI checks that every source URL resolves without moving hundreds of megabytes # on each of the six matrix legs. Run them with `pytest -m slow`. -addopts = "-m 'not slow'" +addopts = "-m 'not slow and not network'" markers = [ "slow: downloads a large dataset; deselected by default", + "network: asks every source server whether it still has the file; run on a schedule", ] [build-system]