diff --git a/README.md b/README.md
index 2f04198..0fb72ff 100644
--- a/README.md
+++ b/README.md
@@ -185,7 +185,7 @@ You can download six files:
| `topic_terms.csv` | One row per topic and term, in long format |
| `topic_similarity.csv` | The cosine similarity of each topic pair |
| `config.json` | The settings that produced the result |
-| `project.zip` | The five files above, plus a `README.txt` |
+| `project.zip` | The five files above, a `README.txt`, and a `figures/` folder with every chart |
## Install
diff --git a/SPECS.md b/SPECS.md
index c7aa56d..2392b81 100644
--- a/SPECS.md
+++ b/SPECS.md
@@ -659,11 +659,21 @@ topic_terms.csv
topic_similarity.csv
config.json
README.txt
+figures/
```
Create the ZIP locally with Python's standard `zipfile` module.
-Optionally expose Vega-Lite chart specs as JSON. PNG/SVG chart download is nice-to-have, not an MVP blocker.
+`figures/` holds every chart of the result view. `exports.figure_files` builds it:
+
+- Each Altair chart is a standalone HTML page. The page holds its data and loads Vega from a CDN.
+- Each word cloud is a PNG.
+- The group, date, and position charts appear only when their data exists.
+- Without *Include the document text*, the document map holds no snippet.
+
+The notebook builds the ZIP only when the reader clicks the button, because the word clouds are slow
+under Pyodide. A PNG or SVG export of an Altair chart is not planned. It needs `vl-convert`, which
+Pyodide does not ship.
### Reproducibility
diff --git a/app.py b/app.py
index a399f64..dd5ee5e 100644
--- a/app.py
+++ b/app.py
@@ -1691,8 +1691,11 @@ def _(AppConfig, display_result, exports, include_text, mo):
# Export the settings of the fitted result, not the live controls. A changed control
# would otherwise describe a run that did not make these files.
_config = AppConfig.model_validate(display_result.config)
+ _include_text = include_text.value
+ # The figures include one word cloud per topic, which is slow under Pyodide. A callable
+ # builds the package only when the reader clicks the button.
_package = mo.download(
- data=exports.project_zip(display_result, _config, include_text.value),
+ data=lambda: exports.project_zip(display_result, _config, _include_text),
filename="project.zip",
label="Download complete research package (.zip)",
mimetype="application/zip",
@@ -1734,7 +1737,8 @@ def _(AppConfig, display_result, exports, include_text, mo):
[
_package,
mo.md(
- "The package holds every table, the settings to repeat the run, and a README."
+ "The package holds every table, every figure, the settings to repeat the run,"
+ " and a README."
),
include_text,
mo.accordion({"Individual files": mo.vstack(_buttons, gap=0.5, align="start")}),
diff --git a/simple_topic_modeling/exports.py b/simple_topic_modeling/exports.py
index 4862132..4cd12cf 100644
--- a/simple_topic_modeling/exports.py
+++ b/simple_topic_modeling/exports.py
@@ -11,9 +11,11 @@
import zipfile
from typing import TYPE_CHECKING, Any
+import altair as alt
import numpy as np
import pandas as pd
+from simple_topic_modeling import plots
from simple_topic_modeling.metrics import topic_similarity
if TYPE_CHECKING:
@@ -25,6 +27,7 @@
"ZIP_README",
"config_json",
"documents_topics_frame",
+ "figure_files",
"project_zip",
"to_csv_bytes",
"topic_similarity_frame",
@@ -55,9 +58,14 @@
topic_terms.csv long format, one row per topic and term
topic_similarity.csv cosine similarity between every pair of topics
config.json the settings that produced this result
+figures/ every chart of the app: HTML for the charts, PNG for the word clouds
Topic scores are shares. Each document's scores sum to 1.
The projection columns place a document on the 2-D map. The axes carry no meaning.
+Near points on a map use similar words. Read a distance as a hint, not as a measure.
+
+Open a figure in a web browser. An HTML figure loads the Vega libraries from cdn.jsdelivr.net,
+so it needs an internet connection. It sends none of your data.
"""
@@ -197,19 +205,97 @@ def config_json(config: AppConfig, topic_names: list[str] | None = None) -> byte
return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False).encode("utf-8")
+class _ScriptSafeEncoder(json.JSONEncoder):
+ r"""Escape `<`, `>`, and `&`, so a JSON value cannot end the `` would otherwise close the block and run as markup in the reader's browser.
+
+ >>> json.dumps({"name": ""}, cls=_ScriptSafeEncoder)
+ '{"name": "\\u003c/script\\u003e"}'
+ """
+
+ def encode(self, o: object) -> str:
+ """Encode `o`, then escape the three characters that HTML can read as markup."""
+ text = super().encode(o)
+ return text.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026")
+
+
+def figure_files(result: TopicModelResult, include_text: bool = False) -> dict[str, bytes]:
+ r"""Render every chart of the result view as a file that opens without the app.
+
+ An Altair chart becomes a standalone HTML page. A word cloud stays a PNG. A chart that needs
+ a metadata column, or the long-text mode, appears only when its data exists. Without
+ `include_text`, the document map holds no snippet, as `documents_topics.csv` holds no text.
+
+ >>> from simple_topic_modeling.result import _example_result
+ >>> files = figure_files(_example_result())
+ >>> list(files)[:5]
+ ['topic_map.html', 'topic_prevalence.html', 'topic_similarity.html', 'document_map.html',
+ 'dominant_topic_score_distribution.html']
+ >>> files["topic_01_wordcloud.png"][:4]
+ b'\x89PNG'
+ """
+ documents = plots.document_frame(result)
+ if not include_text:
+ documents = documents.drop(columns=["snippet"])
+ scores = pd.DataFrame({"score": result.dominant_topic_score})
+ charts: dict[str, Any] = {
+ "topic_map.html": plots.topic_map(result),
+ "topic_prevalence.html": plots.prevalence_bars(result),
+ "topic_similarity.html": plots.similarity_heatmap(result),
+ "document_map.html": plots.document_scatter(documents),
+ "dominant_topic_score_distribution.html": plots.score_histogram(scores),
+ }
+ columns = result.metadata.columns
+ if "group" in columns:
+ charts["group_shares.html"] = plots.group_stacked_bars(
+ plots.group_share_frame(result, "group")
+ )
+ if "date" in columns:
+ parsed, unparsed = plots.parse_dates(result.metadata["date"])
+ if len(parsed) > unparsed:
+ frame = plots.time_share_frame(result, parsed, plots.choose_date_bin(parsed))
+ charts["topic_shares_over_time.html"] = plots.time_line_chart(frame)
+ positions = None
+ if result.config.get("analyse_as") == "long_document":
+ positions = plots.position_frame(result)
+ charts["topic_positions.html"] = plots.position_heatmap(positions, result.topic_names)
+ for topic in range(result.n_topics):
+ prefix = f"topic_{topic + 1:02d}"
+ charts[f"{prefix}_top_terms.html"] = plots.top_term_bars(result, topic)
+ if positions is not None:
+ charts[f"{prefix}_positions.html"] = plots.topic_position_area(positions, topic)
+
+ # The default transformer writes the data into the page, so the file needs no server. A
+ # long text gives a position chart above the default limit of 5,000 rows. Altair types
+ # `PluginEnabler.__exit__` without `None`, so ty rejects a valid `with`.
+ with alt.data_transformers.enable("default", max_rows=None): # ty: ignore[invalid-context-manager]
+ files = {
+ name: chart.to_html(json_kwds={"cls": _ScriptSafeEncoder}).encode("utf-8")
+ for name, chart in charts.items()
+ }
+ for topic in range(result.n_topics):
+ files[f"topic_{topic + 1:02d}_wordcloud.png"] = plots.word_cloud_png(result, topic)
+ return files
+
+
def project_zip(result: TopicModelResult, config: AppConfig, include_text: bool = False) -> bytes:
"""Bundle every export into `project.zip`.
- `SPECS.md` section 7 names the six entries. The bundle is built in memory with the standard
- `zipfile` module.
+ `SPECS.md` section 7 names the six entries and the `figures/` folder. The bundle is built in
+ memory with the standard `zipfile` module.
>>> import zipfile, io
>>> from simple_topic_modeling.config import AppConfig
>>> from simple_topic_modeling.result import _example_result
>>> data = project_zip(_example_result(), AppConfig())
- >>> zipfile.ZipFile(io.BytesIO(data)).namelist()
+ >>> names = zipfile.ZipFile(io.BytesIO(data)).namelist()
+ >>> [name for name in names if not name.startswith("figures/")]
['documents_topics.csv', 'topics.csv', 'topic_terms.csv',
'topic_similarity.csv', 'config.json', 'README.txt']
+ >>> "figures/topic_map.html" in names
+ True
"""
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
@@ -221,4 +307,6 @@ def project_zip(result: TopicModelResult, config: AppConfig, include_text: bool
archive.writestr("topic_similarity.csv", to_csv_bytes(topic_similarity_frame(result)))
archive.writestr("config.json", config_json(config, result.topic_names))
archive.writestr("README.txt", ZIP_README)
+ for name, data in figure_files(result, include_text).items():
+ archive.writestr(f"figures/{name}", data)
return buffer.getvalue()
diff --git a/simple_topic_modeling/plots.py b/simple_topic_modeling/plots.py
index 93c929e..eca2f6a 100644
--- a/simple_topic_modeling/plots.py
+++ b/simple_topic_modeling/plots.py
@@ -498,8 +498,10 @@ def document_scatter(frame: pd.DataFrame, sample_limit: int = DOCUMENT_SAMPLE_LI
alt.Tooltip("document_id:N", title="Document"),
alt.Tooltip("topic:N", title="Topic"),
alt.Tooltip("score:Q", title="Score", format=".3f"),
- alt.Tooltip("snippet:N", title="Text"),
]
+ # An export without the document text drops the snippet column, and the tooltip follows.
+ if "snippet" in frame.columns:
+ tooltip.append(alt.Tooltip("snippet:N", title="Text"))
return (
alt.Chart(frame, title="Document map")
.mark_circle(size=60, opacity=0.6)
diff --git a/tests/test_exports.py b/tests/test_exports.py
index 500e0b2..6943c70 100644
--- a/tests/test_exports.py
+++ b/tests/test_exports.py
@@ -2,6 +2,7 @@
import json
import zipfile
+import altair as alt
import numpy as np
import pandas as pd
import pytest
@@ -11,13 +12,14 @@
EXPORT_LABELS,
config_json,
documents_topics_frame,
+ figure_files,
project_zip,
to_csv_bytes,
topic_similarity_frame,
topic_terms_frame,
topics_frame,
)
-from simple_topic_modeling.io import build_corpus
+from simple_topic_modeling.io import build_corpus, split_long_document
from simple_topic_modeling.modeling import fit_topic_model
from simple_topic_modeling.result import rename_topic
@@ -132,7 +134,8 @@ def test_csv_bytes_are_utf8():
def test_zip_holds_the_six_documented_entries(result):
archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig())))
- assert archive.namelist() == [
+ names = [name for name in archive.namelist() if not name.startswith("figures/")]
+ assert names == [
"documents_topics.csv",
"topics.csv",
"topic_terms.csv",
@@ -158,4 +161,90 @@ def test_zip_carries_the_text_when_asked(result):
def test_every_single_file_has_a_task_label(result):
archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig())))
- assert set(EXPORT_LABELS) == set(archive.namelist()) - {"README.txt"}
+ files = {name for name in archive.namelist() if not name.startswith("figures/")}
+ assert set(EXPORT_LABELS) == files - {"README.txt"}
+
+
+def test_zip_holds_every_figure_of_a_plain_corpus(result):
+ archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig())))
+ figures = {name for name in archive.namelist() if name.startswith("figures/")}
+ per_topic = {
+ f"figures/topic_{topic:02d}_{kind}"
+ for topic in range(1, result.n_topics + 1)
+ for kind in ("top_terms.html", "wordcloud.png")
+ }
+ base = {
+ "figures/topic_map.html",
+ "figures/topic_prevalence.html",
+ "figures/topic_similarity.html",
+ "figures/document_map.html",
+ "figures/dominant_topic_score_distribution.html",
+ }
+ assert figures == base | per_topic
+
+
+def test_an_html_figure_carries_its_data(result):
+ page = figure_files(result)["topic_map.html"].decode("utf-8")
+ assert "vega-embed" in page
+ assert '"datasets"' in page
+ assert '"number"' in page
+
+
+def test_the_document_map_holds_text_only_when_asked(result):
+ assert b'"snippet"' not in figure_files(result)["document_map.html"]
+ assert b'"snippet"' in figure_files(result, include_text=True)["document_map.html"]
+
+
+def _fit_with_metadata(dates):
+ texts = [
+ "cat dog runs fast",
+ "cat sleeps warm couch",
+ "dog barks postman loudly",
+ "bird sings morning song",
+ "bird flies above trees",
+ "fish swims cold water",
+ ]
+ metadata = pd.DataFrame({"group": ["a", "a", "b", "b", "c", "c"], "date": dates})
+ corpus, _ = build_corpus(texts, [f"d{index}" for index in range(len(texts))], metadata)
+ return fit_topic_model(corpus, AppConfig(model=ModelConfig(n_topics=2, min_df=1)))
+
+
+def test_metadata_figures_appear_when_their_data_exists():
+ dates = ["2024-01-05", "2024-02-10", "2024-03-15", "2024-04-20", "2024-05-25", "2024-06-30"]
+ names = set(figure_files(_fit_with_metadata(dates)))
+ assert {"group_shares.html", "topic_shares_over_time.html"} <= names
+
+
+def test_unreadable_dates_give_no_time_figure():
+ names = set(figure_files(_fit_with_metadata(["soon"] * 6)))
+ assert "group_shares.html" in names
+ assert "topic_shares_over_time.html" not in names
+
+
+def test_a_long_text_adds_the_position_figures():
+ text = "\n\n".join(
+ ["cat dog runs fast", "cat sleeps warm couch", "bird sings morning song"] * 4
+ )
+ segments, identifiers, metadata = split_long_document(text, "book.txt")
+ corpus, _ = build_corpus(segments, identifiers, metadata)
+ config = AppConfig(model=ModelConfig(n_topics=2, min_df=1), analyse_as="long_document")
+ names = set(figure_files(fit_topic_model(corpus, config)))
+ assert {"topic_positions.html", "topic_01_positions.html", "topic_02_positions.html"} <= names
+
+
+def test_figures_ignore_the_row_limit(result, monkeypatch):
+ # Altair refuses more than 5,000 rows by default. A long text exceeds that limit.
+ big = alt.Chart(pd.DataFrame({"x": range(5001)})).mark_point()
+ monkeypatch.setattr("simple_topic_modeling.plots.score_histogram", lambda frame: big)
+ page = figure_files(result)["dominant_topic_score_distribution.html"]
+ assert b'"datasets"' in page
+
+
+def test_a_figure_escapes_a_name_that_closes_the_script(result):
+ attack = "
"
+ files = figure_files(rename_topic(result, 0, attack))
+ for name, data in files.items():
+ if name.endswith(".html"):
+ assert attack.encode("utf-8") not in data
+ assert data.count(b"") == data.count(b"