diff --git a/README.md b/README.md index 2f04198..0fb72ff 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ You can download six files: | `topic_terms.csv` | One row per topic and term, in long format | | `topic_similarity.csv` | The cosine similarity of each topic pair | | `config.json` | The settings that produced the result | -| `project.zip` | The five files above, plus a `README.txt` | +| `project.zip` | The five files above, a `README.txt`, and a `figures/` folder with every chart | ## Install diff --git a/SPECS.md b/SPECS.md index c7aa56d..2392b81 100644 --- a/SPECS.md +++ b/SPECS.md @@ -659,11 +659,21 @@ topic_terms.csv topic_similarity.csv config.json README.txt +figures/ ``` Create the ZIP locally with Python's standard `zipfile` module. -Optionally expose Vega-Lite chart specs as JSON. PNG/SVG chart download is nice-to-have, not an MVP blocker. +`figures/` holds every chart of the result view. `exports.figure_files` builds it: + +- Each Altair chart is a standalone HTML page. The page holds its data and loads Vega from a CDN. +- Each word cloud is a PNG. +- The group, date, and position charts appear only when their data exists. +- Without *Include the document text*, the document map holds no snippet. + +The notebook builds the ZIP only when the reader clicks the button, because the word clouds are slow +under Pyodide. A PNG or SVG export of an Altair chart is not planned. It needs `vl-convert`, which +Pyodide does not ship. ### Reproducibility diff --git a/app.py b/app.py index a399f64..dd5ee5e 100644 --- a/app.py +++ b/app.py @@ -1691,8 +1691,11 @@ def _(AppConfig, display_result, exports, include_text, mo): # Export the settings of the fitted result, not the live controls. A changed control # would otherwise describe a run that did not make these files. _config = AppConfig.model_validate(display_result.config) + _include_text = include_text.value + # The figures include one word cloud per topic, which is slow under Pyodide. A callable + # builds the package only when the reader clicks the button. _package = mo.download( - data=exports.project_zip(display_result, _config, include_text.value), + data=lambda: exports.project_zip(display_result, _config, _include_text), filename="project.zip", label="Download complete research package (.zip)", mimetype="application/zip", @@ -1734,7 +1737,8 @@ def _(AppConfig, display_result, exports, include_text, mo): [ _package, mo.md( - "The package holds every table, the settings to repeat the run, and a README." + "The package holds every table, every figure, the settings to repeat the run," + " and a README." ), include_text, mo.accordion({"Individual files": mo.vstack(_buttons, gap=0.5, align="start")}), diff --git a/simple_topic_modeling/exports.py b/simple_topic_modeling/exports.py index 4862132..4cd12cf 100644 --- a/simple_topic_modeling/exports.py +++ b/simple_topic_modeling/exports.py @@ -11,9 +11,11 @@ import zipfile from typing import TYPE_CHECKING, Any +import altair as alt import numpy as np import pandas as pd +from simple_topic_modeling import plots from simple_topic_modeling.metrics import topic_similarity if TYPE_CHECKING: @@ -25,6 +27,7 @@ "ZIP_README", "config_json", "documents_topics_frame", + "figure_files", "project_zip", "to_csv_bytes", "topic_similarity_frame", @@ -55,9 +58,14 @@ topic_terms.csv long format, one row per topic and term topic_similarity.csv cosine similarity between every pair of topics config.json the settings that produced this result +figures/ every chart of the app: HTML for the charts, PNG for the word clouds Topic scores are shares. Each document's scores sum to 1. The projection columns place a document on the 2-D map. The axes carry no meaning. +Near points on a map use similar words. Read a distance as a hint, not as a measure. + +Open a figure in a web browser. An HTML figure loads the Vega libraries from cdn.jsdelivr.net, +so it needs an internet connection. It sends none of your data. """ @@ -197,19 +205,97 @@ def config_json(config: AppConfig, topic_names: list[str] | None = None) -> byte return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False).encode("utf-8") +class _ScriptSafeEncoder(json.JSONEncoder): + r"""Escape `<`, `>`, and `&`, so a JSON value cannot end the `` would otherwise close the block and run as markup in the reader's browser. + + >>> json.dumps({"name": ""}, cls=_ScriptSafeEncoder) + '{"name": "\\u003c/script\\u003e"}' + """ + + def encode(self, o: object) -> str: + """Encode `o`, then escape the three characters that HTML can read as markup.""" + text = super().encode(o) + return text.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + + +def figure_files(result: TopicModelResult, include_text: bool = False) -> dict[str, bytes]: + r"""Render every chart of the result view as a file that opens without the app. + + An Altair chart becomes a standalone HTML page. A word cloud stays a PNG. A chart that needs + a metadata column, or the long-text mode, appears only when its data exists. Without + `include_text`, the document map holds no snippet, as `documents_topics.csv` holds no text. + + >>> from simple_topic_modeling.result import _example_result + >>> files = figure_files(_example_result()) + >>> list(files)[:5] + ['topic_map.html', 'topic_prevalence.html', 'topic_similarity.html', 'document_map.html', + 'dominant_topic_score_distribution.html'] + >>> files["topic_01_wordcloud.png"][:4] + b'\x89PNG' + """ + documents = plots.document_frame(result) + if not include_text: + documents = documents.drop(columns=["snippet"]) + scores = pd.DataFrame({"score": result.dominant_topic_score}) + charts: dict[str, Any] = { + "topic_map.html": plots.topic_map(result), + "topic_prevalence.html": plots.prevalence_bars(result), + "topic_similarity.html": plots.similarity_heatmap(result), + "document_map.html": plots.document_scatter(documents), + "dominant_topic_score_distribution.html": plots.score_histogram(scores), + } + columns = result.metadata.columns + if "group" in columns: + charts["group_shares.html"] = plots.group_stacked_bars( + plots.group_share_frame(result, "group") + ) + if "date" in columns: + parsed, unparsed = plots.parse_dates(result.metadata["date"]) + if len(parsed) > unparsed: + frame = plots.time_share_frame(result, parsed, plots.choose_date_bin(parsed)) + charts["topic_shares_over_time.html"] = plots.time_line_chart(frame) + positions = None + if result.config.get("analyse_as") == "long_document": + positions = plots.position_frame(result) + charts["topic_positions.html"] = plots.position_heatmap(positions, result.topic_names) + for topic in range(result.n_topics): + prefix = f"topic_{topic + 1:02d}" + charts[f"{prefix}_top_terms.html"] = plots.top_term_bars(result, topic) + if positions is not None: + charts[f"{prefix}_positions.html"] = plots.topic_position_area(positions, topic) + + # The default transformer writes the data into the page, so the file needs no server. A + # long text gives a position chart above the default limit of 5,000 rows. Altair types + # `PluginEnabler.__exit__` without `None`, so ty rejects a valid `with`. + with alt.data_transformers.enable("default", max_rows=None): # ty: ignore[invalid-context-manager] + files = { + name: chart.to_html(json_kwds={"cls": _ScriptSafeEncoder}).encode("utf-8") + for name, chart in charts.items() + } + for topic in range(result.n_topics): + files[f"topic_{topic + 1:02d}_wordcloud.png"] = plots.word_cloud_png(result, topic) + return files + + def project_zip(result: TopicModelResult, config: AppConfig, include_text: bool = False) -> bytes: """Bundle every export into `project.zip`. - `SPECS.md` section 7 names the six entries. The bundle is built in memory with the standard - `zipfile` module. + `SPECS.md` section 7 names the six entries and the `figures/` folder. The bundle is built in + memory with the standard `zipfile` module. >>> import zipfile, io >>> from simple_topic_modeling.config import AppConfig >>> from simple_topic_modeling.result import _example_result >>> data = project_zip(_example_result(), AppConfig()) - >>> zipfile.ZipFile(io.BytesIO(data)).namelist() + >>> names = zipfile.ZipFile(io.BytesIO(data)).namelist() + >>> [name for name in names if not name.startswith("figures/")] ['documents_topics.csv', 'topics.csv', 'topic_terms.csv', 'topic_similarity.csv', 'config.json', 'README.txt'] + >>> "figures/topic_map.html" in names + True """ buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: @@ -221,4 +307,6 @@ def project_zip(result: TopicModelResult, config: AppConfig, include_text: bool archive.writestr("topic_similarity.csv", to_csv_bytes(topic_similarity_frame(result))) archive.writestr("config.json", config_json(config, result.topic_names)) archive.writestr("README.txt", ZIP_README) + for name, data in figure_files(result, include_text).items(): + archive.writestr(f"figures/{name}", data) return buffer.getvalue() diff --git a/simple_topic_modeling/plots.py b/simple_topic_modeling/plots.py index 93c929e..eca2f6a 100644 --- a/simple_topic_modeling/plots.py +++ b/simple_topic_modeling/plots.py @@ -498,8 +498,10 @@ def document_scatter(frame: pd.DataFrame, sample_limit: int = DOCUMENT_SAMPLE_LI alt.Tooltip("document_id:N", title="Document"), alt.Tooltip("topic:N", title="Topic"), alt.Tooltip("score:Q", title="Score", format=".3f"), - alt.Tooltip("snippet:N", title="Text"), ] + # An export without the document text drops the snippet column, and the tooltip follows. + if "snippet" in frame.columns: + tooltip.append(alt.Tooltip("snippet:N", title="Text")) return ( alt.Chart(frame, title="Document map") .mark_circle(size=60, opacity=0.6) diff --git a/tests/test_exports.py b/tests/test_exports.py index 500e0b2..6943c70 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -2,6 +2,7 @@ import json import zipfile +import altair as alt import numpy as np import pandas as pd import pytest @@ -11,13 +12,14 @@ EXPORT_LABELS, config_json, documents_topics_frame, + figure_files, project_zip, to_csv_bytes, topic_similarity_frame, topic_terms_frame, topics_frame, ) -from simple_topic_modeling.io import build_corpus +from simple_topic_modeling.io import build_corpus, split_long_document from simple_topic_modeling.modeling import fit_topic_model from simple_topic_modeling.result import rename_topic @@ -132,7 +134,8 @@ def test_csv_bytes_are_utf8(): def test_zip_holds_the_six_documented_entries(result): archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig()))) - assert archive.namelist() == [ + names = [name for name in archive.namelist() if not name.startswith("figures/")] + assert names == [ "documents_topics.csv", "topics.csv", "topic_terms.csv", @@ -158,4 +161,90 @@ def test_zip_carries_the_text_when_asked(result): def test_every_single_file_has_a_task_label(result): archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig()))) - assert set(EXPORT_LABELS) == set(archive.namelist()) - {"README.txt"} + files = {name for name in archive.namelist() if not name.startswith("figures/")} + assert set(EXPORT_LABELS) == files - {"README.txt"} + + +def test_zip_holds_every_figure_of_a_plain_corpus(result): + archive = zipfile.ZipFile(io.BytesIO(project_zip(result, AppConfig()))) + figures = {name for name in archive.namelist() if name.startswith("figures/")} + per_topic = { + f"figures/topic_{topic:02d}_{kind}" + for topic in range(1, result.n_topics + 1) + for kind in ("top_terms.html", "wordcloud.png") + } + base = { + "figures/topic_map.html", + "figures/topic_prevalence.html", + "figures/topic_similarity.html", + "figures/document_map.html", + "figures/dominant_topic_score_distribution.html", + } + assert figures == base | per_topic + + +def test_an_html_figure_carries_its_data(result): + page = figure_files(result)["topic_map.html"].decode("utf-8") + assert "vega-embed" in page + assert '"datasets"' in page + assert '"number"' in page + + +def test_the_document_map_holds_text_only_when_asked(result): + assert b'"snippet"' not in figure_files(result)["document_map.html"] + assert b'"snippet"' in figure_files(result, include_text=True)["document_map.html"] + + +def _fit_with_metadata(dates): + texts = [ + "cat dog runs fast", + "cat sleeps warm couch", + "dog barks postman loudly", + "bird sings morning song", + "bird flies above trees", + "fish swims cold water", + ] + metadata = pd.DataFrame({"group": ["a", "a", "b", "b", "c", "c"], "date": dates}) + corpus, _ = build_corpus(texts, [f"d{index}" for index in range(len(texts))], metadata) + return fit_topic_model(corpus, AppConfig(model=ModelConfig(n_topics=2, min_df=1))) + + +def test_metadata_figures_appear_when_their_data_exists(): + dates = ["2024-01-05", "2024-02-10", "2024-03-15", "2024-04-20", "2024-05-25", "2024-06-30"] + names = set(figure_files(_fit_with_metadata(dates))) + assert {"group_shares.html", "topic_shares_over_time.html"} <= names + + +def test_unreadable_dates_give_no_time_figure(): + names = set(figure_files(_fit_with_metadata(["soon"] * 6))) + assert "group_shares.html" in names + assert "topic_shares_over_time.html" not in names + + +def test_a_long_text_adds_the_position_figures(): + text = "\n\n".join( + ["cat dog runs fast", "cat sleeps warm couch", "bird sings morning song"] * 4 + ) + segments, identifiers, metadata = split_long_document(text, "book.txt") + corpus, _ = build_corpus(segments, identifiers, metadata) + config = AppConfig(model=ModelConfig(n_topics=2, min_df=1), analyse_as="long_document") + names = set(figure_files(fit_topic_model(corpus, config))) + assert {"topic_positions.html", "topic_01_positions.html", "topic_02_positions.html"} <= names + + +def test_figures_ignore_the_row_limit(result, monkeypatch): + # Altair refuses more than 5,000 rows by default. A long text exceeds that limit. + big = alt.Chart(pd.DataFrame({"x": range(5001)})).mark_point() + monkeypatch.setattr("simple_topic_modeling.plots.score_histogram", lambda frame: big) + page = figure_files(result)["dominant_topic_score_distribution.html"] + assert b'"datasets"' in page + + +def test_a_figure_escapes_a_name_that_closes_the_script(result): + attack = "" + files = figure_files(rename_topic(result, 0, attack)) + for name, data in files.items(): + if name.endswith(".html"): + assert attack.encode("utf-8") not in data + assert data.count(b"") == data.count(b"