From 35eea5b4f24cfcbcd9349dfc47ea7e25cc6814d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20M=C3=A4hr?= <14755525+maehr@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:02:31 +0200 Subject: [PATCH 1/2] feat: add the figures to project.zip exports.figure_files renders every chart of the result view. An Altair chart becomes a standalone HTML page with its data inside. A word cloud stays a PNG. The group, date, and position charts appear only when their data exists. Without the document text, the document map holds no snippet, as the CSV holds no text. The download button builds the package only on a click, because the word clouds are slow under Pyodide. Closes #50. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EsEUVKgZPLky1kHXnU2aBW --- README.md | 2 +- SPECS.md | 12 ++++- app.py | 8 ++- simple_topic_modeling/exports.py | 75 ++++++++++++++++++++++++++-- simple_topic_modeling/plots.py | 4 +- tests/test_exports.py | 85 ++++++++++++++++++++++++++++++-- 6 files changed, 175 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2f04198..0fb72ff 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ You can download six files: | `topic_terms.csv` | One row per topic and term, in long format | | `topic_similarity.csv` | The cosine similarity of each topic pair | | `config.json` | The settings that produced the result | -| `project.zip` | The five files above, plus a `README.txt` | +| `project.zip` | The five files above, a `README.txt`, and a `figures/` folder with every chart | ## Install diff --git a/SPECS.md b/SPECS.md index c7aa56d..2392b81 100644 --- a/SPECS.md +++ b/SPECS.md @@ -659,11 +659,21 @@ topic_terms.csv topic_similarity.csv config.json README.txt +figures/ ``` Create the ZIP locally with Python's standard `zipfile` module. -Optionally expose Vega-Lite chart specs as JSON. PNG/SVG chart download is nice-to-have, not an MVP blocker. +`figures/` holds every chart of the result view. `exports.figure_files` builds it: + +- Each Altair chart is a standalone HTML page. The page holds its data and loads Vega from a CDN. +- Each word cloud is a PNG. +- The group, date, and position charts appear only when their data exists. +- Without *Include the document text*, the document map holds no snippet. + +The notebook builds the ZIP only when the reader clicks the button, because the word clouds are slow +under Pyodide. A PNG or SVG export of an Altair chart is not planned. It needs `vl-convert`, which +Pyodide does not ship. ### Reproducibility diff --git a/app.py b/app.py index a399f64..dd5ee5e 100644 --- a/app.py +++ b/app.py @@ -1691,8 +1691,11 @@ def _(AppConfig, display_result, exports, include_text, mo): # Export the settings of the fitted result, not the live controls. A changed control # would otherwise describe a run that did not make these files. _config = AppConfig.model_validate(display_result.config) + _include_text = include_text.value + # The figures include one word cloud per topic, which is slow under Pyodide. A callable + # builds the package only when the reader clicks the button. _package = mo.download( - data=exports.project_zip(display_result, _config, include_text.value), + data=lambda: exports.project_zip(display_result, _config, _include_text), filename="project.zip", label="Download complete research package (.zip)", mimetype="application/zip", @@ -1734,7 +1737,8 @@ def _(AppConfig, display_result, exports, include_text, mo): [ _package, mo.md( - "The package holds every table, the settings to repeat the run, and a README." + "The package holds every table, every figure, the settings to repeat the run," + " and a README." ), include_text, mo.accordion({"Individual files": mo.vstack(_buttons, gap=0.5, align="start")}), diff --git a/simple_topic_modeling/exports.py b/simple_topic_modeling/exports.py index 4862132..db20422 100644 --- a/simple_topic_modeling/exports.py +++ b/simple_topic_modeling/exports.py @@ -11,9 +11,11 @@ import zipfile from typing import TYPE_CHECKING, Any +import altair as alt import numpy as np import pandas as pd +from simple_topic_modeling import plots from simple_topic_modeling.metrics import topic_similarity if TYPE_CHECKING: @@ -25,6 +27,7 @@ "ZIP_README", "config_json", "documents_topics_frame", + "figure_files", "project_zip", "to_csv_bytes", "topic_similarity_frame", @@ -55,9 +58,14 @@ topic_terms.csv long format, one row per topic and term topic_similarity.csv cosine similarity between every pair of topics config.json the settings that produced this result +figures/ every chart of the app: HTML for the charts, PNG for the word clouds Topic scores are shares. Each document's scores sum to 1. The projection columns place a document on the 2-D map. The axes carry no meaning. +Near points on a map use similar words. Read a distance as a hint, not as a measure. + +Open a figure in a web browser. An HTML figure loads the Vega libraries from cdn.jsdelivr.net, +so it needs an internet connection. It sends none of your data. """ @@ -197,19 +205,78 @@ def config_json(config: AppConfig, topic_names: list[str] | None = None) -> byte return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False).encode("utf-8") +def figure_files(result: TopicModelResult, include_text: bool = False) -> dict[str, bytes]: + r"""Render every chart of the result view as a file that opens without the app. + + An Altair chart becomes a standalone HTML page. A word cloud stays a PNG. A chart that needs + a metadata column, or the long-text mode, appears only when its data exists. Without + `include_text`, the document map holds no snippet, as `documents_topics.csv` holds no text. + + >>> from simple_topic_modeling.result import _example_result + >>> files = figure_files(_example_result()) + >>> list(files)[:5] + ['topic_map.html', 'topic_prevalence.html', 'topic_similarity.html', 'document_map.html', + 'dominant_topic_score_distribution.html'] + >>> files["topic_01_wordcloud.png"][:4] + b'\x89PNG' + """ + documents = plots.document_frame(result) + if not include_text: + documents = documents.drop(columns=["snippet"]) + scores = pd.DataFrame({"score": result.dominant_topic_score}) + charts: dict[str, Any] = { + "topic_map.html": plots.topic_map(result), + "topic_prevalence.html": plots.prevalence_bars(result), + "topic_similarity.html": plots.similarity_heatmap(result), + "document_map.html": plots.document_scatter(documents), + "dominant_topic_score_distribution.html": plots.score_histogram(scores), + } + columns = result.metadata.columns + if "group" in columns: + charts["group_shares.html"] = plots.group_stacked_bars( + plots.group_share_frame(result, "group") + ) + if "date" in columns: + parsed, unparsed = plots.parse_dates(result.metadata["date"]) + if len(parsed) > unparsed: + frame = plots.time_share_frame(result, parsed, plots.choose_date_bin(parsed)) + charts["topic_shares_over_time.html"] = plots.time_line_chart(frame) + positions = None + if result.config.get("analyse_as") == "long_document": + positions = plots.position_frame(result) + charts["topic_positions.html"] = plots.position_heatmap(positions, result.topic_names) + for topic in range(result.n_topics): + prefix = f"topic_{topic + 1:02d}" + charts[f"{prefix}_top_terms.html"] = plots.top_term_bars(result, topic) + if positions is not None: + charts[f"{prefix}_positions.html"] = plots.topic_position_area(positions, topic) + + # The default transformer writes the data into the page, so the file needs no server. A + # long text gives a position chart above the default limit of 5,000 rows. Altair types + # `PluginEnabler.__exit__` without `None`, so ty rejects a valid `with`. + with alt.data_transformers.enable("default", max_rows=None): # ty: ignore[invalid-context-manager] + files = {name: chart.to_html().encode("utf-8") for name, chart in charts.items()} + for topic in range(result.n_topics): + files[f"topic_{topic + 1:02d}_wordcloud.png"] = plots.word_cloud_png(result, topic) + return files + + def project_zip(result: TopicModelResult, config: AppConfig, include_text: bool = False) -> bytes: """Bundle every export into `project.zip`. - `SPECS.md` section 7 names the six entries. The bundle is built in memory with the standard - `zipfile` module. + `SPECS.md` section 7 names the six entries and the `figures/` folder. The bundle is built in + memory with the standard `zipfile` module. >>> import zipfile, io >>> from simple_topic_modeling.config import AppConfig >>> from simple_topic_modeling.result import _example_result >>> data = project_zip(_example_result(), AppConfig()) - >>> zipfile.ZipFile(io.BytesIO(data)).namelist() + >>> names = zipfile.ZipFile(io.BytesIO(data)).namelist() + >>> [name for name in names if not name.startswith("figures/")] ['documents_topics.csv', 'topics.csv', 'topic_terms.csv', 'topic_similarity.csv', 'config.json', 'README.txt'] + >>> "figures/topic_map.html" in names + True """ buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: @@ -221,4 +288,6 @@ def project_zip(result: TopicModelResult, config: AppConfig, include_text: bool archive.writestr("topic_similarity.csv", to_csv_bytes(topic_similarity_frame(result))) archive.writestr("config.json", config_json(config, result.topic_names)) archive.writestr("README.txt", ZIP_README) + for name, data in figure_files(result, include_text).items(): + archive.writestr(f"figures/{name}", data) return buffer.getvalue() diff --git a/simple_topic_modeling/plots.py b/simple_topic_modeling/plots.py index 93c929e..eca2f6a 100644 --- a/simple_topic_modeling/plots.py +++ b/simple_topic_modeling/plots.py @@ -498,8 +498,10 @@ def document_scatter(frame: pd.DataFrame, sample_limit: int = DOCUMENT_SAMPLE_LI alt.Tooltip("document_id:N", title="Document"), alt.Tooltip("topic:N", title="Topic"), alt.Tooltip("score:Q", title="Score", format=".3f"), - alt.Tooltip("snippet:N", title="Text"), ] + # An export without the document text drops the snippet column, and the tooltip follows. + if "snippet" in frame.columns: + tooltip.append(alt.Tooltip("snippet:N", title="Text")) return ( alt.Chart(frame, title="Document map") .mark_circle(size=60, opacity=0.6) diff --git a/tests/test_exports.py b/tests/test_exports.py index 500e0b2..98d6492 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -2,6 +2,7 @@ import json import zipfile +import altair as alt import numpy as np import pandas as pd import pytest @@ -11,13 +12,14 @@ EXPORT_LABELS, config_json, documents_topics_frame, + figure_files, project_zip, to_csv_bytes, topic_similarity_frame, topic_terms_frame, topics_frame, ) -from simple_topic_modeling.io import build_corpus +from simple_topic_modeling.io import build_corpus, split_long_document from simple_topic_modeling.modeling import fit_topic_model from simple_topic_modeling.result import rename_topic @@ -132,7 +134,8 @@ def test_csv_bytes_are_utf8(): def test_zip_holds_the_six_documented_entries(result): archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig()))) - assert archive.namelist() == [ + names = [name for name in archive.namelist() if not name.startswith("figures/")] + assert names == [ "documents_topics.csv", "topics.csv", "topic_terms.csv", @@ -158,4 +161,80 @@ def test_zip_carries_the_text_when_asked(result): def test_every_single_file_has_a_task_label(result): archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig()))) - assert set(EXPORT_LABELS) == set(archive.namelist()) - {"README.txt"} + files = {name for name in archive.namelist() if not name.startswith("figures/")} + assert set(EXPORT_LABELS) == files - {"README.txt"} + + +def test_zip_holds_every_figure_of_a_plain_corpus(result): + archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig()))) + figures = {name for name in archive.namelist() if name.startswith("figures/")} + per_topic = { + f"figures/topic_{topic:02d}_{kind}" + for topic in range(1, result.n_topics + 1) + for kind in ("top_terms.html", "wordcloud.png") + } + base = { + "figures/topic_map.html", + "figures/topic_prevalence.html", + "figures/topic_similarity.html", + "figures/document_map.html", + "figures/dominant_topic_score_distribution.html", + } + assert figures == base | per_topic + + +def test_an_html_figure_carries_its_data(result): + page = figure_files(result)["topic_map.html"].decode("utf-8") + assert "vega-embed" in page + assert '"datasets"' in page + assert '"number"' in page + + +def test_the_document_map_holds_text_only_when_asked(result): + assert b'"snippet"' not in figure_files(result)["document_map.html"] + assert b'"snippet"' in figure_files(result, include_text=True)["document_map.html"] + + +def _fit_with_metadata(dates): + texts = [ + "cat dog runs fast", + "cat sleeps warm couch", + "dog barks postman loudly", + "bird sings morning song", + "bird flies above trees", + "fish swims cold water", + ] + metadata = pd.DataFrame({"group": ["a", "a", "b", "b", "c", "c"], "date": dates}) + corpus, _ = build_corpus(texts, [f"d{index}" for index in range(len(texts))], metadata) + return fit_topic_model(corpus, AppConfig(model=ModelConfig(n_topics=2, min_df=1))) + + +def test_metadata_figures_appear_when_their_data_exists(): + dates = ["2024-01-05", "2024-02-10", "2024-03-15", "2024-04-20", "2024-05-25", "2024-06-30"] + names = set(figure_files(_fit_with_metadata(dates))) + assert {"group_shares.html", "topic_shares_over_time.html"} <= names + + +def test_unreadable_dates_give_no_time_figure(): + names = set(figure_files(_fit_with_metadata(["soon"] * 6))) + assert "group_shares.html" in names + assert "topic_shares_over_time.html" not in names + + +def test_a_long_text_adds_the_position_figures(): + text = "\n\n".join( + ["cat dog runs fast", "cat sleeps warm couch", "bird sings morning song"] * 4 + ) + segments, identifiers, metadata = split_long_document(text, "book.txt") + corpus, _ = build_corpus(segments, identifiers, metadata) + config = AppConfig(model=ModelConfig(n_topics=2, min_df=1), analyse_as="long_document") + names = set(figure_files(fit_topic_model(corpus, config))) + assert {"topic_positions.html", "topic_01_positions.html", "topic_02_positions.html"} <= names + + +def test_figures_ignore_the_row_limit(result, monkeypatch): + # Altair refuses more than 5,000 rows by default. A long text exceeds that limit. + big = alt.Chart(pd.DataFrame({"x": range(5001)})).mark_point() + monkeypatch.setattr("simple_topic_modeling.plots.score_histogram", lambda frame: big) + page = figure_files(result)["dominant_topic_score_distribution.html"] + assert b'"datasets"' in page From 83791e9ebd03ebac51303a498494f0d9d02cbe42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20M=C3=A4hr?= <14755525+maehr@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:35:40 +0200 Subject: [PATCH 2/2] fix: escape the chart JSON in an exported figure Altair writes the chart JSON into a script block without escaping. A topic name, a document ID, or a snippet with would close the block and run as markup when a reader opens the figure. The export now escapes <, >, and & as JSON unicode escapes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EsEUVKgZPLky1kHXnU2aBW --- simple_topic_modeling/exports.py | 21 ++++++++++++++++++++- tests/test_exports.py | 10 ++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/simple_topic_modeling/exports.py b/simple_topic_modeling/exports.py index db20422..4cd12cf 100644 --- a/simple_topic_modeling/exports.py +++ b/simple_topic_modeling/exports.py @@ -205,6 +205,22 @@ def config_json(config: AppConfig, topic_names: list[str] | None = None) -> byte return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False).encode("utf-8") +class _ScriptSafeEncoder(json.JSONEncoder): + r"""Escape `<`, `>`, and `&`, so a JSON value cannot end the `` would otherwise close the block and run as markup in the reader's browser. + + >>> json.dumps({"name": ""}, cls=_ScriptSafeEncoder) + '{"name": "\\u003c/script\\u003e"}' + """ + + def encode(self, o: object) -> str: + """Encode `o`, then escape the three characters that HTML can read as markup.""" + text = super().encode(o) + return text.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + + def figure_files(result: TopicModelResult, include_text: bool = False) -> dict[str, bytes]: r"""Render every chart of the result view as a file that opens without the app. @@ -255,7 +271,10 @@ def figure_files(result: TopicModelResult, include_text: bool = False) -> dict[s # long text gives a position chart above the default limit of 5,000 rows. Altair types # `PluginEnabler.__exit__` without `None`, so ty rejects a valid `with`. with alt.data_transformers.enable("default", max_rows=None): # ty: ignore[invalid-context-manager] - files = {name: chart.to_html().encode("utf-8") for name, chart in charts.items()} + files = { + name: chart.to_html(json_kwds={"cls": _ScriptSafeEncoder}).encode("utf-8") + for name, chart in charts.items() + } for topic in range(result.n_topics): files[f"topic_{topic + 1:02d}_wordcloud.png"] = plots.word_cloud_png(result, topic) return files diff --git a/tests/test_exports.py b/tests/test_exports.py index 98d6492..6943c70 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -238,3 +238,13 @@ def test_figures_ignore_the_row_limit(result, monkeypatch): monkeypatch.setattr("simple_topic_modeling.plots.score_histogram", lambda frame: big) page = figure_files(result)["dominant_topic_score_distribution.html"] assert b'"datasets"' in page + + +def test_a_figure_escapes_a_name_that_closes_the_script(result): + attack = "" + files = figure_files(rename_topic(result, 0, attack)) + for name, data in files.items(): + if name.endswith(".html"): + assert attack.encode("utf-8") not in data + assert data.count(b"") == data.count(b"