feat: add the figures to project.zip

README.md

@@ -185,7 +185,7 @@ You can download six files:
 | `topic_terms.csv` | One row per topic and term, in long format |
 | `topic_similarity.csv` | The cosine similarity of each topic pair |
 | `config.json` | The settings that produced the result |
-| `project.zip` | The five files above, plus a `README.txt` |
+| `project.zip` | The five files above, a `README.txt`, and a `figures/` folder with every chart |
 
 ## Install
 

SPECS.md

@@ -659,11 +659,21 @@ topic_terms.csv
 topic_similarity.csv
 config.json
 README.txt
+figures/
 ```
 
 Create the ZIP locally with Python's standard `zipfile` module.
 
-Optionally expose Vega-Lite chart specs as JSON. PNG/SVG chart download is nice-to-have, not an MVP blocker.
+`figures/` holds every chart of the result view. `exports.figure_files` builds it:
+
+- Each Altair chart is a standalone HTML page. The page holds its data and loads Vega from a CDN.
+- Each word cloud is a PNG.
+- The group, date, and position charts appear only when their data exists.
+- Without *Include the document text*, the document map holds no snippet.
+
+The notebook builds the ZIP only when the reader clicks the button, because the word clouds are slow
+under Pyodide. A PNG or SVG export of an Altair chart is not planned. It needs `vl-convert`, which
+Pyodide does not ship.
 
 ### Reproducibility
 

app.py

@@ -1691,8 +1691,11 @@ def _(AppConfig, display_result, exports, include_text, mo):
         # Export the settings of the fitted result, not the live controls. A changed control
         # would otherwise describe a run that did not make these files.
         _config = AppConfig.model_validate(display_result.config)
+        _include_text = include_text.value
+        # The figures include one word cloud per topic, which is slow under Pyodide. A callable
+        # builds the package only when the reader clicks the button.
         _package = mo.download(
-            data=exports.project_zip(display_result, _config, include_text.value),
+            data=lambda: exports.project_zip(display_result, _config, _include_text),
             filename="project.zip",
             label="Download complete research package (.zip)",
             mimetype="application/zip",
@@ -1734,7 +1737,8 @@ def _(AppConfig, display_result, exports, include_text, mo):
             [
                 _package,
                 mo.md(
-                    "The package holds every table, the settings to repeat the run, and a README."
+                    "The package holds every table, every figure, the settings to repeat the run,"
+                    " and a README."
                 ),
                 include_text,
                 mo.accordion({"Individual files": mo.vstack(_buttons, gap=0.5, align="start")}),

simple_topic_modeling/exports.py

@@ -11,9 +11,11 @@
 import zipfile
 from typing import TYPE_CHECKING, Any
 
+import altair as alt
 import numpy as np
 import pandas as pd
 
+from simple_topic_modeling import plots
 from simple_topic_modeling.metrics import topic_similarity
 
 if TYPE_CHECKING:
@@ -25,6 +27,7 @@
     "ZIP_README",
     "config_json",
     "documents_topics_frame",
+    "figure_files",
     "project_zip",
     "to_csv_bytes",
     "topic_similarity_frame",
@@ -55,9 +58,14 @@
 topic_terms.csv        long format, one row per topic and term
 topic_similarity.csv   cosine similarity between every pair of topics
 config.json            the settings that produced this result
+figures/               every chart of the app: HTML for the charts, PNG for the word clouds
 
 Topic scores are shares. Each document's scores sum to 1.
 The projection columns place a document on the 2-D map. The axes carry no meaning.
+Near points on a map use similar words. Read a distance as a hint, not as a measure.
+
+Open a figure in a web browser. An HTML figure loads the Vega libraries from cdn.jsdelivr.net,
+so it needs an internet connection. It sends none of your data.
 """
 
 
@@ -197,19 +205,97 @@ def config_json(config: AppConfig, topic_names: list[str] | None = None) -> byte
     return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False).encode("utf-8")
 
 
+class _ScriptSafeEncoder(json.JSONEncoder):
+    r"""Escape `<`, `>`, and `&`, so a JSON value cannot end the `<script>` block of a page.
+
+    Altair writes the chart JSON into the page as it is. A topic name or a document ID such as
+    `</script>` would otherwise close the block and run as markup in the reader's browser.
+
+    >>> json.dumps({"name": "</script>"}, cls=_ScriptSafeEncoder)
+    '{"name": "\\u003c/script\\u003e"}'
+    """
+
+    def encode(self, o: object) -> str:
+        """Encode `o`, then escape the three characters that HTML can read as markup."""
+        text = super().encode(o)
+        return text.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026")
+
+
+def figure_files(result: TopicModelResult, include_text: bool = False) -> dict[str, bytes]:
+    r"""Render every chart of the result view as a file that opens without the app.
+
+    An Altair chart becomes a standalone HTML page. A word cloud stays a PNG. A chart that needs
+    a metadata column, or the long-text mode, appears only when its data exists. Without
+    `include_text`, the document map holds no snippet, as `documents_topics.csv` holds no text.
+
+    >>> from simple_topic_modeling.result import _example_result
+    >>> files = figure_files(_example_result())
+    >>> list(files)[:5]
+    ['topic_map.html', 'topic_prevalence.html', 'topic_similarity.html', 'document_map.html',
+     'dominant_topic_score_distribution.html']
+    >>> files["topic_01_wordcloud.png"][:4]
+    b'\x89PNG'
+    """
+    documents = plots.document_frame(result)
+    if not include_text:
+        documents = documents.drop(columns=["snippet"])
+    scores = pd.DataFrame({"score": result.dominant_topic_score})
+    charts: dict[str, Any] = {
+        "topic_map.html": plots.topic_map(result),
+        "topic_prevalence.html": plots.prevalence_bars(result),
+        "topic_similarity.html": plots.similarity_heatmap(result),
+        "document_map.html": plots.document_scatter(documents),
+        "dominant_topic_score_distribution.html": plots.score_histogram(scores),
+    }
+    columns = result.metadata.columns
+    if "group" in columns:
+        charts["group_shares.html"] = plots.group_stacked_bars(
+            plots.group_share_frame(result, "group")
+        )
+    if "date" in columns:
+        parsed, unparsed = plots.parse_dates(result.metadata["date"])
+        if len(parsed) > unparsed:
+            frame = plots.time_share_frame(result, parsed, plots.choose_date_bin(parsed))
+            charts["topic_shares_over_time.html"] = plots.time_line_chart(frame)
+    positions = None
+    if result.config.get("analyse_as") == "long_document":
+        positions = plots.position_frame(result)
+        charts["topic_positions.html"] = plots.position_heatmap(positions, result.topic_names)
+    for topic in range(result.n_topics):
+        prefix = f"topic_{topic + 1:02d}"
+        charts[f"{prefix}_top_terms.html"] = plots.top_term_bars(result, topic)
+        if positions is not None:
+            charts[f"{prefix}_positions.html"] = plots.topic_position_area(positions, topic)
+
+    # The default transformer writes the data into the page, so the file needs no server. A
+    # long text gives a position chart above the default limit of 5,000 rows. Altair types
+    # `PluginEnabler.__exit__` without `None`, so ty rejects a valid `with`.
+    with alt.data_transformers.enable("default", max_rows=None):  # ty: ignore[invalid-context-manager]
+        files = {
+            name: chart.to_html(json_kwds={"cls": _ScriptSafeEncoder}).encode("utf-8")
+            for name, chart in charts.items()
+        }
+    for topic in range(result.n_topics):
+        files[f"topic_{topic + 1:02d}_wordcloud.png"] = plots.word_cloud_png(result, topic)
+    return files
+
+
 def project_zip(result: TopicModelResult, config: AppConfig, include_text: bool = False) -> bytes:
     """Bundle every export into `project.zip`.
 
-    `SPECS.md` section 7 names the six entries. The bundle is built in memory with the standard
-    `zipfile` module.
+    `SPECS.md` section 7 names the six entries and the `figures/` folder. The bundle is built in
+    memory with the standard `zipfile` module.
 
     >>> import zipfile, io
     >>> from simple_topic_modeling.config import AppConfig
     >>> from simple_topic_modeling.result import _example_result
     >>> data = project_zip(_example_result(), AppConfig())
-    >>> zipfile.ZipFile(io.BytesIO(data)).namelist()
+    >>> names = zipfile.ZipFile(io.BytesIO(data)).namelist()
+    >>> [name for name in names if not name.startswith("figures/")]
     ['documents_topics.csv', 'topics.csv', 'topic_terms.csv',
      'topic_similarity.csv', 'config.json', 'README.txt']
+    >>> "figures/topic_map.html" in names
+    True
     """
     buffer = io.BytesIO()
     with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
@@ -221,4 +307,6 @@ def project_zip(result: TopicModelResult, config: AppConfig, include_text: bool
         archive.writestr("topic_similarity.csv", to_csv_bytes(topic_similarity_frame(result)))
         archive.writestr("config.json", config_json(config, result.topic_names))
         archive.writestr("README.txt", ZIP_README)
+        for name, data in figure_files(result, include_text).items():
+            archive.writestr(f"figures/{name}", data)
     return buffer.getvalue()

simple_topic_modeling/plots.py

@@ -498,8 +498,10 @@ def document_scatter(frame: pd.DataFrame, sample_limit: int = DOCUMENT_SAMPLE_LI
         alt.Tooltip("document_id:N", title="Document"),
         alt.Tooltip("topic:N", title="Topic"),
         alt.Tooltip("score:Q", title="Score", format=".3f"),
-        alt.Tooltip("snippet:N", title="Text"),
     ]
+    # An export without the document text drops the snippet column, and the tooltip follows.
+    if "snippet" in frame.columns:
+        tooltip.append(alt.Tooltip("snippet:N", title="Text"))
     return (
         alt.Chart(frame, title="Document map")
         .mark_circle(size=60, opacity=0.6)

tests/test_exports.py

@@ -2,6 +2,7 @@
 import json
 import zipfile
 
+import altair as alt
 import numpy as np
 import pandas as pd
 import pytest
@@ -11,13 +12,14 @@
     EXPORT_LABELS,
     config_json,
     documents_topics_frame,
+    figure_files,
     project_zip,
     to_csv_bytes,
     topic_similarity_frame,
     topic_terms_frame,
     topics_frame,
 )
-from simple_topic_modeling.io import build_corpus
+from simple_topic_modeling.io import build_corpus, split_long_document
 from simple_topic_modeling.modeling import fit_topic_model
 from simple_topic_modeling.result import rename_topic
 
@@ -132,7 +134,8 @@ def test_csv_bytes_are_utf8():
 
 def test_zip_holds_the_six_documented_entries(result):
     archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig())))
-    assert archive.namelist() == [
+    names = [name for name in archive.namelist() if not name.startswith("figures/")]
+    assert names == [
         "documents_topics.csv",
         "topics.csv",
         "topic_terms.csv",
@@ -158,4 +161,90 @@ def test_zip_carries_the_text_when_asked(result):
 
 def test_every_single_file_has_a_task_label(result):
     archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig())))
-    assert set(EXPORT_LABELS) == set(archive.namelist()) - {"README.txt"}
+    files = {name for name in archive.namelist() if not name.startswith("figures/")}
+    assert set(EXPORT_LABELS) == files - {"README.txt"}
+
+
+def test_zip_holds_every_figure_of_a_plain_corpus(result):
+    archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig())))
+    figures = {name for name in archive.namelist() if name.startswith("figures/")}
+    per_topic = {
+        f"figures/topic_{topic:02d}_{kind}"
+        for topic in range(1, result.n_topics + 1)
+        for kind in ("top_terms.html", "wordcloud.png")
+    }
+    base = {
+        "figures/topic_map.html",
+        "figures/topic_prevalence.html",
+        "figures/topic_similarity.html",
+        "figures/document_map.html",
+        "figures/dominant_topic_score_distribution.html",
+    }
+    assert figures == base | per_topic
+
+
+def test_an_html_figure_carries_its_data(result):
+    page = figure_files(result)["topic_map.html"].decode("utf-8")
+    assert "vega-embed" in page
+    assert '"datasets"' in page
+    assert '"number"' in page
+
+
+def test_the_document_map_holds_text_only_when_asked(result):
+    assert b'"snippet"' not in figure_files(result)["document_map.html"]
+    assert b'"snippet"' in figure_files(result, include_text=True)["document_map.html"]
+
+
+def _fit_with_metadata(dates):
+    texts = [
+        "cat dog runs fast",
+        "cat sleeps warm couch",
+        "dog barks postman loudly",
+        "bird sings morning song",
+        "bird flies above trees",
+        "fish swims cold water",
+    ]
+    metadata = pd.DataFrame({"group": ["a", "a", "b", "b", "c", "c"], "date": dates})
+    corpus, _ = build_corpus(texts, [f"d{index}" for index in range(len(texts))], metadata)
+    return fit_topic_model(corpus, AppConfig(model=ModelConfig(n_topics=2, min_df=1)))
+
+
+def test_metadata_figures_appear_when_their_data_exists():
+    dates = ["2024-01-05", "2024-02-10", "2024-03-15", "2024-04-20", "2024-05-25", "2024-06-30"]
+    names = set(figure_files(_fit_with_metadata(dates)))
+    assert {"group_shares.html", "topic_shares_over_time.html"} <= names
+
+
+def test_unreadable_dates_give_no_time_figure():
+    names = set(figure_files(_fit_with_metadata(["soon"] * 6)))
+    assert "group_shares.html" in names
+    assert "topic_shares_over_time.html" not in names
+
+
+def test_a_long_text_adds_the_position_figures():
+    text = "\n\n".join(
+        ["cat dog runs fast", "cat sleeps warm couch", "bird sings morning song"] * 4
+    )
+    segments, identifiers, metadata = split_long_document(text, "book.txt")
+    corpus, _ = build_corpus(segments, identifiers, metadata)
+    config = AppConfig(model=ModelConfig(n_topics=2, min_df=1), analyse_as="long_document")
+    names = set(figure_files(fit_topic_model(corpus, config)))
+    assert {"topic_positions.html", "topic_01_positions.html", "topic_02_positions.html"} <= names
+
+
+def test_figures_ignore_the_row_limit(result, monkeypatch):
+    # Altair refuses more than 5,000 rows by default. A long text exceeds that limit.
+    big = alt.Chart(pd.DataFrame({"x": range(5001)})).mark_point()
+    monkeypatch.setattr("simple_topic_modeling.plots.score_histogram", lambda frame: big)
+    page = figure_files(result)["dominant_topic_score_distribution.html"]
+    assert b'"datasets"' in page
+
+
+def test_a_figure_escapes_a_name_that_closes_the_script(result):
+    attack = "</script><img src=x onerror=alert(1)>"
+    files = figure_files(rename_topic(result, 0, attack))
+    for name, data in files.items():
+        if name.endswith(".html"):
+            assert attack.encode("utf-8") not in data
+            assert data.count(b"</script>") == data.count(b"<script")
+    assert b"\\u003c/script\\u003e" in files["topic_map.html"]