From 6682558ee8e7ef35c908d0060117562bd9854d57 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Thu, 10 Sep 2026 15:03:25 -0500 Subject: [PATCH 1/2] fix(docs): repair Python API cross-references --- ci/scripts/build-docs.sh | 2 + ci/scripts/check_python_doc_crossrefs.py | 99 ++++++++++++++++++++ docs/README.md | 14 +++ mkdocs.yml | 1 + python/sedonadb/python/sedonadb/context.py | 8 +- python/sedonadb/python/sedonadb/dataframe.py | 23 +++-- python/sedonadb/python/sedonadb/read.py | 6 +- 7 files changed, 139 insertions(+), 14 deletions(-) create mode 100644 ci/scripts/check_python_doc_crossrefs.py diff --git a/ci/scripts/build-docs.sh b/ci/scripts/build-docs.sh index a2d972ae4b..226c44cffd 100755 --- a/ci/scripts/build-docs.sh +++ b/ci/scripts/build-docs.sh @@ -56,6 +56,8 @@ popd pushd "${SEDONADB_DIR}" if mkdocs build --strict ; then + python3 ci/scripts/check_python_doc_crossrefs.py \ + site/reference/python/index.html || exit 1 echo "Success!" exit 0 else diff --git a/ci/scripts/check_python_doc_crossrefs.py b/ci/scripts/check_python_doc_crossrefs.py new file mode 100644 index 0000000000..9496c8b477 --- /dev/null +++ b/ci/scripts/check_python_doc_crossrefs.py @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Check that representative Python API cross-references rendered as links.""" + +from __future__ import annotations + +import sys +from html.parser import HTMLParser +from pathlib import Path + +EXPECTED_LINKS = { + ("SedonaContext", "#sedonadb.context.SedonaContext"), + ("connect()", "#sedonadb.context.connect"), + ( + "create_data_frame()", + "#sedonadb.context.SedonaContext.create_data_frame", + ), + ("read_parquet()", "#sedonadb.context.SedonaContext.read_parquet"), + ("read_pyogrio()", "#sedonadb.context.SedonaContext.read_pyogrio"), + ("sql()", "#sedonadb.context.SedonaContext.sql"), + ("select()", "#sedonadb.dataframe.DataFrame.select"), + ("filter()", "#sedonadb.dataframe.DataFrame.filter"), + ("sort()", "#sedonadb.dataframe.DataFrame.sort"), + ("limit()", "#sedonadb.dataframe.DataFrame.limit"), + ("to_view()", "#sedonadb.dataframe.DataFrame.to_view"), +} + + +class LinkCollector(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.links: set[tuple[str, str]] = set() + self.visible_text: list[str] = [] + self._href: str | None = None + self._link_text: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + self._href = dict(attrs).get("href") + self._link_text = [] + + def handle_data(self, data: str) -> None: + self.visible_text.append(data) + if self._href is not None: + self._link_text.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag == "a" and self._href is not None: + text = " ".join("".join(self._link_text).split()) + self.links.add((text, self._href)) + self._href = None + self._link_text = [] + + +def main() -> int: + if len(sys.argv) != 2: + print(f"Usage: {Path(sys.argv[0]).name} PYTHON_REFERENCE_HTML", file=sys.stderr) + return 2 + + html_path = Path(sys.argv[1]) + parser = LinkCollector() + parser.feed(html_path.read_text(encoding="utf-8")) + + missing = sorted(EXPECTED_LINKS - parser.links) + if missing: + print("Python API cross-references did not render:", file=sys.stderr) + for text, href in missing: + print(f" {text!r} -> {href}", file=sys.stderr) + return 1 + + visible_text = "".join(parser.visible_text) + if "][sedonadb." in visible_text: + print( + "Unrendered Python API cross-reference markup found in generated HTML", + file=sys.stderr, + ) + return 1 + + print(f"Validated {len(EXPECTED_LINKS)} Python API cross-references") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/README.md b/docs/README.md index 1aba55d927..6718c95fc4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -44,6 +44,20 @@ When iterating on documentation, it is usually best to use the `mkdocs` commands * `mkdocs build` - Build the documentation site. * `mkdocs -h` - Print help message and exit. +## Python API cross-references + +Python docstrings are rendered by mkdocstrings. Link to another documented Python +object with an autorefs reference-style link whose target is the object's fully +qualified name: + +```markdown +[`DataFrame`][sedonadb.dataframe.DataFrame] +``` + +Backticks alone (for example, `` `DataFrame` ``) only format text as code and do +not create a link. Sphinx roles such as `` :class:`DataFrame` `` are not supported +by the Markdown docstring renderer. + The official documentation is built using a script which may be useful when building the documentation locally for the first time: diff --git a/mkdocs.yml b/mkdocs.yml index 0fbb99de87..7addb612d2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -138,6 +138,7 @@ markdown_extensions: - pymdownx.tilde plugins: - search + - autorefs - macros - git-revision-date-localized: type: datetime diff --git a/python/sedonadb/python/sedonadb/context.py b/python/sedonadb/python/sedonadb/context.py index 4dd046668d..5ad92aa976 100644 --- a/python/sedonadb/python/sedonadb/context.py +++ b/python/sedonadb/python/sedonadb/context.py @@ -176,7 +176,7 @@ def create_data_frame(self, obj: Any, schema: Any = None) -> DataFrame: return _create_data_frame(self, obj, schema) def view(self, name: str) -> DataFrame: - """Create a [DataFrame][sedonadb.dataframe.DataFrame] from a named view + """Create a [`DataFrame`][sedonadb.dataframe.DataFrame] from a named view Refer to a named view registered with this context. @@ -228,7 +228,7 @@ def read_parquet( validate: bool = False, partitioning: Union[str, Iterable[str], None] = None, ) -> DataFrame: - """Create a [DataFrame][sedonadb.dataframe.DataFrame] from one or more Parquet files + """Create a [`DataFrame`][sedonadb.dataframe.DataFrame] from one or more Parquet files Args: table_paths: A str, Path, or iterable of paths containing URLs to Parquet @@ -381,7 +381,7 @@ def read_pyogrio( def sql( self, sql: str, *, params: Union[List, Tuple, Dict, None] = None ) -> DataFrame: - """Create a [DataFrame][sedonadb.dataframe.DataFrame] by executing SQL + """Create a [`DataFrame`][sedonadb.dataframe.DataFrame] by executing SQL Parses a SQL string into a logical plan and returns a DataFrame that can be used to request results or further modify the query. @@ -582,7 +582,7 @@ def lit(self, value: Any) -> LiteralExpr: def connect() -> SedonaContext: - """Create a new [SedonaContext][sedonadb.context.SedonaContext] + """Create a new [`SedonaContext`][sedonadb.context.SedonaContext] Runtime configuration (memory limits, spill directory, pool type) can be set via `options` on the returned context before executing diff --git a/python/sedonadb/python/sedonadb/dataframe.py b/python/sedonadb/python/sedonadb/dataframe.py index bc4247e283..044936a50f 100644 --- a/python/sedonadb/python/sedonadb/dataframe.py +++ b/python/sedonadb/python/sedonadb/dataframe.py @@ -34,13 +34,22 @@ class DataFrame: """Representation of a (lazy) collection of columns - This object is usually constructed from `sd = sedona.db.connect()` - by importing an object with `sd.create_data_frame()`, reading a file - with `sd.read_parquet()`/`sd.read_pyogrio()`, or executing SQL with - `sd.sql()`. Once created, a DataFrame can be modified using the Python - API (e.g., `.select()`, `.filter()`, `.sort()`, `.limit()`) or by - creating a temporary view with `.to_view("name")` and querying the - resulting view using `sd.sql()`. The Python API aims to provide + This object is usually constructed from a + [`SedonaContext`][sedonadb.context.SedonaContext], returned by + [`connect()`][sedonadb.context.connect], by importing an object with + [`create_data_frame()`][sedonadb.context.SedonaContext.create_data_frame], + reading a file with + [`read_parquet()`][sedonadb.context.SedonaContext.read_parquet] or + [`read_pyogrio()`][sedonadb.context.SedonaContext.read_pyogrio], or executing + SQL with [`sql()`][sedonadb.context.SedonaContext.sql]. Once created, a + DataFrame can be modified using the Python API (e.g., + [`select()`][sedonadb.dataframe.DataFrame.select], + [`filter()`][sedonadb.dataframe.DataFrame.filter], + [`sort()`][sedonadb.dataframe.DataFrame.sort], or + [`limit()`][sedonadb.dataframe.DataFrame.limit]) or by creating a temporary + view with [`to_view()`][sedonadb.dataframe.DataFrame.to_view] and querying + the resulting view using + [`sql()`][sedonadb.context.SedonaContext.sql]. The Python API aims to provide a minimal subset of functionality derived primarily from Ibis and DuckDB's relational APIs. diff --git a/python/sedonadb/python/sedonadb/read.py b/python/sedonadb/python/sedonadb/read.py index 664adfc8fb..48e6373282 100644 --- a/python/sedonadb/python/sedonadb/read.py +++ b/python/sedonadb/python/sedonadb/read.py @@ -184,7 +184,7 @@ def parquet( validate: bool = False, partitioning: Union[str, Iterable[str], None] = None, ) -> DataFrame: - """Create a [DataFrame][sedonadb.dataframe.DataFrame] from one or more Parquet files + """Create a [`DataFrame`][sedonadb.dataframe.DataFrame] from one or more Parquet files Args: table_paths: A str, Path, or iterable of paths containing URLs to Parquet @@ -275,7 +275,7 @@ def csv( has_header: bool = True, delimiter: str = ",", ) -> DataFrame: - """Create a [DataFrame][sedonadb.dataframe.DataFrame] from one or more CSV files. + """Create a [`DataFrame`][sedonadb.dataframe.DataFrame] from one or more CSV files. The schema is inferred from the file(s). Geometry is not inferred; parse WKT/WKB columns explicitly (e.g. `ST_GeomFromText`) after reading. @@ -317,7 +317,7 @@ def json( table_paths: Union[str, Path, Iterable[str]], options: Optional[Dict[str, Any]] = None, ) -> DataFrame: - """Create a [DataFrame][sedonadb.dataframe.DataFrame] from newline-delimited JSON. + """Create a [`DataFrame`][sedonadb.dataframe.DataFrame] from newline-delimited JSON. Reads newline-delimited JSON (NDJSON / JSON Lines) — one JSON object per line — not a single JSON array. The schema is inferred. From 9b6f109f1d5c21a18d2e5ca5f6f774a3dceb2ba0 Mon Sep 17 00:00:00 2001 From: Dewey Dunnington Date: Fri, 11 Sep 2026 10:29:54 -0500 Subject: [PATCH 2/2] chore(docs): rely on documented docs build --- ci/scripts/build-docs.sh | 2 - ci/scripts/check_python_doc_crossrefs.py | 99 ------------------------ 2 files changed, 101 deletions(-) delete mode 100644 ci/scripts/check_python_doc_crossrefs.py diff --git a/ci/scripts/build-docs.sh b/ci/scripts/build-docs.sh index 226c44cffd..a2d972ae4b 100755 --- a/ci/scripts/build-docs.sh +++ b/ci/scripts/build-docs.sh @@ -56,8 +56,6 @@ popd pushd "${SEDONADB_DIR}" if mkdocs build --strict ; then - python3 ci/scripts/check_python_doc_crossrefs.py \ - site/reference/python/index.html || exit 1 echo "Success!" exit 0 else diff --git a/ci/scripts/check_python_doc_crossrefs.py b/ci/scripts/check_python_doc_crossrefs.py deleted file mode 100644 index 9496c8b477..0000000000 --- a/ci/scripts/check_python_doc_crossrefs.py +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Check that representative Python API cross-references rendered as links.""" - -from __future__ import annotations - -import sys -from html.parser import HTMLParser -from pathlib import Path - -EXPECTED_LINKS = { - ("SedonaContext", "#sedonadb.context.SedonaContext"), - ("connect()", "#sedonadb.context.connect"), - ( - "create_data_frame()", - "#sedonadb.context.SedonaContext.create_data_frame", - ), - ("read_parquet()", "#sedonadb.context.SedonaContext.read_parquet"), - ("read_pyogrio()", "#sedonadb.context.SedonaContext.read_pyogrio"), - ("sql()", "#sedonadb.context.SedonaContext.sql"), - ("select()", "#sedonadb.dataframe.DataFrame.select"), - ("filter()", "#sedonadb.dataframe.DataFrame.filter"), - ("sort()", "#sedonadb.dataframe.DataFrame.sort"), - ("limit()", "#sedonadb.dataframe.DataFrame.limit"), - ("to_view()", "#sedonadb.dataframe.DataFrame.to_view"), -} - - -class LinkCollector(HTMLParser): - def __init__(self) -> None: - super().__init__() - self.links: set[tuple[str, str]] = set() - self.visible_text: list[str] = [] - self._href: str | None = None - self._link_text: list[str] = [] - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - if tag == "a": - self._href = dict(attrs).get("href") - self._link_text = [] - - def handle_data(self, data: str) -> None: - self.visible_text.append(data) - if self._href is not None: - self._link_text.append(data) - - def handle_endtag(self, tag: str) -> None: - if tag == "a" and self._href is not None: - text = " ".join("".join(self._link_text).split()) - self.links.add((text, self._href)) - self._href = None - self._link_text = [] - - -def main() -> int: - if len(sys.argv) != 2: - print(f"Usage: {Path(sys.argv[0]).name} PYTHON_REFERENCE_HTML", file=sys.stderr) - return 2 - - html_path = Path(sys.argv[1]) - parser = LinkCollector() - parser.feed(html_path.read_text(encoding="utf-8")) - - missing = sorted(EXPECTED_LINKS - parser.links) - if missing: - print("Python API cross-references did not render:", file=sys.stderr) - for text, href in missing: - print(f" {text!r} -> {href}", file=sys.stderr) - return 1 - - visible_text = "".join(parser.visible_text) - if "][sedonadb." in visible_text: - print( - "Unrendered Python API cross-reference markup found in generated HTML", - file=sys.stderr, - ) - return 1 - - print(f"Validated {len(EXPECTED_LINKS)} Python API cross-references") - return 0 - - -if __name__ == "__main__": - sys.exit(main())