From 9beff314f2c439916c9a59ce26aa77af2d2b0cbb Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Fri, 25 Sep 2026 14:08:31 -0300 Subject: [PATCH 01/13] fix(models): build INTERNAL results that carry a column the model does not declare The API adds columns without an SDK release, and each one made every INTERNAL call of its resource raise ParseError. `_to_fields` is now the one place that keys an answer by field: it leaves out the keys that name no column or field of the model, logs them once per answer at DEBUG, and every single-request INTERNAL resource builds through it. An answer that is not an object, or that carries none of the model's columns, comes back as it is, so the model still refuses it. The CSV merge of stocks.candles and options.quotes keeps such a column, as every other CSV does, and refuses only a header with none of the resource's columns. --- CHANGELOG.md | 3 +- src/marketdata/output_types/columns.py | 31 ++++- .../output_types/stocks_earnings.py | 15 ++- src/marketdata/output_types/stocks_prices.py | 15 ++- src/marketdata/output_types/stocks_quotes.py | 15 ++- src/marketdata/resources/funds/candles.py | 3 +- src/marketdata/resources/markets/status.py | 3 +- src/marketdata/resources/options/chain.py | 4 +- .../resources/options/expirations.py | 4 +- src/marketdata/resources/options/lookup.py | 3 +- src/marketdata/resources/stocks/news.py | 5 +- src/marketdata/resources/stocks/prices.py | 7 +- src/marketdata/resources/stocks/quotes.py | 8 +- src/marketdata/resources/utilities/base.py | 3 +- src/marketdata/utils.py | 22 +-- src/tests/test_money_decimal.py | 127 +++++++++++++++--- src/tests/test_options_quotes.py | 17 +++ src/tests/test_stocks_candles.py | 21 +++ src/tests/test_utils.py | 11 ++ 19 files changed, 261 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0096678..ec60c31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Money in the `OutputFormat.INTERNAL` models is `decimal.Decimal` instead of `float`, built from the digits in the response body, which those resources now decode with `parse_float=Decimal` on the INTERNAL path: `ask`, `bid`, `mid`, `last`, `change` of `StockQuote`, `mid` and `change` of `StockPrice`, `o`, `h`, `l`, `c` of `StockCandle` and `FundsCandle`, the three EPS lists of `StockEarnings`, `strike`, `bid`, `mid`, `ask`, `last`, `intrinsicValue`, `extrinsicValue`, `underlyingPrice` of `OptionsChain` and `OptionsQuotes`, and the same fields of every human-readable twin. Greeks, IV and percentages stay `float`, sizes and counts `int`; DataFrames (pandas and polars), JSON and CSV output keep the plain float parse. Migration: arithmetic mixing `Decimal` and `float` raises `TypeError`, and a comparison with a float literal compares against its binary value (`Decimal("65.1") == 65.1` is `False`), so use `Decimal` literals or `int`; `float(value)` gives a float back, and `OutputFormat.DATAFRAME` is the float path for analysis (a pandas DataFrame built from models gets `object` columns, and `json.dumps(..., default=str)` writes the money as strings). A model built by hand converts its money arguments: a float or a numpy scalar through its shortest repr (`65.1` is `Decimal("65.1")`), a NaN number, which is how pandas writes a missing value, to `None`; a value that is not a number, a string that reads as NaN, or an infinity raises and names the field, while an array (anything that is not a string and has `__array__` and `__len__`: a numpy array, a pandas or polars Series, a pandas Index) is left as it came, and a `dict`, a `set` or a generator raises. On the API path the same refusal is a `ParseError` naming the field and the body, not a bare `TypeError` (#50) - Dates on `OutputFormat.INTERNAL` are read with the API's own rule and always come back as US/Eastern datetimes. A number from 10000 up to 200000 is a spreadsheet serial of US/Eastern wall-clock time, from 200000 Unix seconds, from `1e10` milliseconds and from `1e13` nanoseconds. A number under 10000 is a relative range for the API, not a date, and raises `ParseError`. The SDK used a boundary of its own at 60000, so from April 2064 a serial would have read as 1970. A `timestamp` date (`2026-09-21`) and a spreadsheet serial are now midnight and wall-clock time in US/Eastern, where they came back as naive datetimes, and a serial is rounded to the second. The intraday `from_date` and `to_date` of `stocks.candles()` are read only when they are ISO dates or numbers the API reads as dates (a spreadsheet serial or a Unix time), which the SDK splits into one request per year: a relative range or a keyword goes to the API as it is, in one request, where `"60"` was sent as 1900-02-28 and `"yesterday"` was refused. `UserRateLimits.reset_time` takes a datetime or Unix seconds, and refuses anything else (#116). Migration: compare INTERNAL dates against aware datetimes (`pytz.timezone("US/Eastern").localize(...)`), not naive ones - A response body carrying a `NaN`, `Infinity` or `-Infinity` literal raises `ParseError` on every output format that decodes it, where `INTERNAL`, `JSON` and `DATAFRAME` used to return it as `nan` or `inf`. Those literals are not JSON, and the renderer behind the API's data endpoints refuses them, answering `500`, so one in their bodies was written by something in between (`/status/` and `/headers/` are served elsewhere). `CSV` output writes the API's body as it came, since it reads no number, except for `client.utilities`, which builds its CSV from the decoded body and refuses it too. A number past what a float holds is not refused: `JSON` and `DATAFRAME` read `1e400` as `inf` while `INTERNAL` keeps it exact, and `INTERNAL` refuses one past what a `Decimal` holds. That renderer does not write such a number, and refusing it on the float path would slow every decode. Migration: code that read `nan` or `inf` from such a body catches `ParseError` (or `BaseMarketdataException`) instead; no setting brings the literals back (#50) -- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a key the model does not take or a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError`, on `stocks.earnings()`, `options.lookup()`, and on `options.chain()` and `options.expirations()` with the API's names; elsewhere it raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) +- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones; elsewhere it raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) - `options.chain(delta=...)` checks a number before it sends it, the way it checks `strike`, and keeps it as the text it sends. A `bool`, a NaN, an infinity, a number a float cannot hold and a delta outside -1 to 1 now raise `ValidationError` before any request, where they used to be sent: `delta=nan` came back as the whole chain, since the API reads it as no filter, `delta=True` was sent as `1.0`, and `delta=30` was sent as `30.0`. `OptionsChainInput.delta` holds that text (`"0.5"`) instead of a `float` (#101). Migration: pass a finite number from -1 to 1, the way the public docs write a delta (`0.3`), a `DeltaFilter`, or `delta=None` for no filter; code that reads `OptionsChainInput(...).delta` gets the text sent to the API, and `float()` turns a plain number back. - Every date field of the `OutputFormat.INTERNAL` models admits `None` in its annotation: `datetime.datetime | None` for one date, `list[datetime.datetime | None]` for a column of them, and `datetime.date | None` for `MarketStatus.date`, as `OptionsExpirations.updated` already did. A `null` date in an answer reads as `None` in its own row (see Fixed), and the annotations now say so. Migration: a type checker now asks for a `None` check before a `datetime` method is called on a date field (`if quote.updated is not None: quote.updated.date()`). At run time nothing that used to return changes: an answer with a `null` date used to raise `ParseError`, or on `OptionsQuotes` shift the column (#106) @@ -57,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `stocks.candles()` and `options.quotes()` retry each request on its own instead of the whole call: a failed chunk or symbol is re-issued alone, the healthy responses are kept, and one unreachable symbol no longer re-sends (and re-bills) every other symbol on each attempt (#83) - `options.expirations()` returns a `no_data` DataFrame with the same index as a populated one (`expirations` index, `updated` column), so concatenating results across symbols keeps the index name and gains no stray `expirations` column (#84) - The CSV output of `stocks.candles()` and `options.quotes()` is merged from the answers themselves: the file keeps the header the API sent (the requested `columns`, the human-readable names) and every row of every chunk or symbol; it used to compare each body against the model's field list and silently drop every body that differed, so a `columns=` request or a human-readable `options.quotes()` produced a header-only file, and an undecodable body (an HTML error page) vanished without a word. Such a body now raises `ParseError`, and so does a body the `csv` module cannot read (a field past its size limit), which escaped as a raw `csv.Error` (#86) +- A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model leaves it out, and its name is logged once per answer at DEBUG on `marketdata.logger`, where the call raised a bare `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` leave such a column out of their INTERNAL, DataFrame and JSON output, as they did, without the DEBUG line. Their CSV output keeps it, as the CSV of every other resource does; a header with none of the resource's columns, like an HTML error page, is still refused (#111) - DataFrame dates under `DateFormat.TIMESTAMP` and `DateFormat.SPREADSHEET` are the US/Eastern datetimes the default format gives, on pandas and polars. polars left `timestamp` datetimes as strings, a `timestamp` date (`2026-09-21`) landed at 20:00 the day before on both, and a spreadsheet serial, which the API writes as US/Eastern wall-clock time, was read as UTC, four or five hours early. A serial now reads to the second, without the float noise pandas kept (#143) - The `no_data` DataFrame and the header-only CSV carry the requested columns under a `columns=` filter, in request order, as a populated answer does; API names select the human-readable twins under `use_human_readable=True`, and under `add_headers=False` the empty CSV is an empty file. Known limit: the API's own aliases (`open` for `o`, `price`, `date`) are not translated, because the accepted set depends on the endpoint, so filtering on one of those still gives an empty frame of a different shape than the populated answer (#87) - A `404` `no_data` whose body is not JSON (a CDN or a proxy answering the 404 without one) returns the empty result on the JSON output too, instead of raising `ParseError` there while the other three formats returned their empty value: the output format never decides whether a call raises (#87) diff --git a/src/marketdata/output_types/columns.py b/src/marketdata/output_types/columns.py index 9e2b770..5cff0dc 100644 --- a/src/marketdata/output_types/columns.py +++ b/src/marketdata/output_types/columns.py @@ -1,8 +1,12 @@ +import logging from dataclasses import fields, is_dataclass from typing import Any from marketdata.utils import column_key +# By name, not `get_logger()`: importing the package must not attach a handler. +logger = logging.getLogger("marketdata.logger") + def _column_names(output_model: type) -> dict[str, str]: """Map each field of an output model to the name of its column in the API's @@ -52,19 +56,32 @@ def _column_types(output_model: type) -> dict[str, Any]: } -def _to_fields(output_model: type, data: dict) -> dict: - """Rename the keys of an API answer from column names to field names. +def _to_fields(output_model: type, data: Any) -> Any: + """Key an API answer by the fields of the output model it is built into. Args: - output_model: The output model the answer is built into. + output_model: The output model dataclass the answer is built into. data: The answer, keyed by column name. Returns: - The same values keyed by field name. A key that names no column is - kept as it is. + The values of the keys that name a column or a field of the model, + keyed by field name. Any other key is left out and logged at DEBUG, + except the status flag ``s``. An answer that is not an object, or that + carries none of the model's columns, comes back as it is, for the + model to refuse. """ - by_column = {column: name for name, column in _column_names(output_model).items()} - return {by_column.get(key, key): value for key, value in data.items()} + columns = _column_names(output_model) + if not isinstance(data, dict) or not any(key in data for key in columns.values()): + return data + by_key = {field.name: field.name for field in fields(output_model)} + by_key.update({column: name for name, column in columns.items()}) + undeclared = [key for key in data if key not in by_key and key != "s"] + if undeclared: + logger.debug( + f"The API sent the columns {undeclared!r}, which" + f" {output_model.__name__} does not declare; they are left out" + ) + return {by_key[key]: value for key, value in data.items() if key in by_key} def _model_columns(output_model: type, requested: list[str] | None = None) -> list[str]: diff --git a/src/marketdata/output_types/stocks_earnings.py b/src/marketdata/output_types/stocks_earnings.py index c9ffc4f..366d4a0 100644 --- a/src/marketdata/output_types/stocks_earnings.py +++ b/src/marketdata/output_types/stocks_earnings.py @@ -69,7 +69,20 @@ def __str__(self) -> str: @classmethod def from_dict(cls, data: dict) -> "StockEarnings": - return cls(**data) + """Build the model from an API answer keyed by column name. + + Args: + data: The answer, keyed as the API names its columns. + + Returns: + The earnings. A column the model does not declare is left out. + + Raises: + TypeError: If a field is missing or a number field holds something + that is not a number. + ValueError: If a value cannot be read as its field's type. + """ + return cls(**_to_fields(cls, data)) @dataclass diff --git a/src/marketdata/output_types/stocks_prices.py b/src/marketdata/output_types/stocks_prices.py index a9789e9..c9fef80 100644 --- a/src/marketdata/output_types/stocks_prices.py +++ b/src/marketdata/output_types/stocks_prices.py @@ -36,7 +36,20 @@ def __str__(self) -> str: @classmethod def from_dict(cls, data: dict) -> "StockPrice": - return cls(**data) + """Build the model from an API answer keyed by column name. + + Args: + data: One price, keyed as the API names its columns. + + Returns: + The price. A column the model does not declare is left out. + + Raises: + TypeError: If a field is missing or a number field holds something + that is not a number. + ValueError: If a value cannot be read as its field's type. + """ + return cls(**_to_fields(cls, data)) @dataclass diff --git a/src/marketdata/output_types/stocks_quotes.py b/src/marketdata/output_types/stocks_quotes.py index 719a93c..74ea015 100644 --- a/src/marketdata/output_types/stocks_quotes.py +++ b/src/marketdata/output_types/stocks_quotes.py @@ -47,7 +47,20 @@ def __str__(self) -> str: @classmethod def from_dict(cls, data: dict) -> "StockQuote": - return cls(**data) + """Build the model from an API answer keyed by column name. + + Args: + data: One quote, keyed as the API names its columns. + + Returns: + The quote. A column the model does not declare is left out. + + Raises: + TypeError: If a field is missing or a number field holds something + that is not a number. + ValueError: If a value cannot be read as its field's type. + """ + return cls(**_to_fields(cls, data)) @dataclass diff --git a/src/marketdata/resources/funds/candles.py b/src/marketdata/resources/funds/candles.py index c3f1230..3fe79f7 100644 --- a/src/marketdata/resources/funds/candles.py +++ b/src/marketdata/resources/funds/candles.py @@ -5,6 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.funds import FundsCandlesInput from marketdata.output_handlers import get_dataframe_output_handler +from marketdata.output_types.columns import _to_fields from marketdata.output_types.funds_candles import FundsCandle, FundsCandlesHumanReadable from marketdata.params import universal_params from marketdata.resources.base import BaseResource, model_errors, no_data_result @@ -69,7 +70,7 @@ def candles( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = parse_json(response, exact=True) - data = get_data_records(data, exclude_keys=["s"]) + data = get_data_records(_to_fields(output_model, data), exclude_keys=["s"]) with model_errors(response): return [output_model(**row) for row in data] diff --git a/src/marketdata/resources/markets/status.py b/src/marketdata/resources/markets/status.py index 55b0f40..f26379f 100644 --- a/src/marketdata/resources/markets/status.py +++ b/src/marketdata/resources/markets/status.py @@ -5,6 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.markets import MarketStatusInput from marketdata.output_handlers import get_dataframe_output_handler +from marketdata.output_types.columns import _to_fields from marketdata.output_types.markets_status import ( MarketStatus, MarketStatusHumanReadable, @@ -66,7 +67,7 @@ def status( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = parse_json(response) - data = get_data_records(data, exclude_keys=["s"]) + data = get_data_records(_to_fields(output_model, data), exclude_keys=["s"]) with model_errors(response): return [output_model(**row) for row in data] diff --git a/src/marketdata/resources/options/chain.py b/src/marketdata/resources/options/chain.py index 35602ab..9cdb90e 100644 --- a/src/marketdata/resources/options/chain.py +++ b/src/marketdata/resources/options/chain.py @@ -69,10 +69,8 @@ def chain( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = parse_json(response, exact=True) - if user_universal_params.use_human_readable: - data = _to_fields(output_model, data) with model_errors(response): - return output_model(**data) + return output_model(**_to_fields(output_model, data)) elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/options/expirations.py b/src/marketdata/resources/options/expirations.py index 824d03a..268419c 100644 --- a/src/marketdata/resources/options/expirations.py +++ b/src/marketdata/resources/options/expirations.py @@ -81,10 +81,8 @@ def expirations( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = parse_json(response) - if user_universal_params.use_human_readable: - data = _to_fields(output_model, data) with model_errors(response): - return output_model(**data) + return output_model(**_to_fields(output_model, data)) elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/options/lookup.py b/src/marketdata/resources/options/lookup.py index 66e9d89..6b7bc10 100644 --- a/src/marketdata/resources/options/lookup.py +++ b/src/marketdata/resources/options/lookup.py @@ -5,6 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.options import OptionsLookupInput from marketdata.output_handlers import get_dataframe_output_handler +from marketdata.output_types.columns import _to_fields from marketdata.output_types.options_lookup import ( OptionsLookup, OptionsLookupHumanReadable, @@ -73,7 +74,7 @@ def lookup( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = parse_json(response) with model_errors(response): - return output_model(**data) + return output_model(**_to_fields(output_model, data)) elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/stocks/news.py b/src/marketdata/resources/stocks/news.py index 547ed88..4a5ef2e 100644 --- a/src/marketdata/resources/stocks/news.py +++ b/src/marketdata/resources/stocks/news.py @@ -5,6 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.stocks import StocksNewsInput from marketdata.output_handlers import get_dataframe_output_handler +from marketdata.output_types.columns import _to_fields from marketdata.output_types.stocks_news import StockNews, StockNewsHumanReadable from marketdata.params import universal_params from marketdata.resources.base import BaseResource, model_errors, no_data_result @@ -68,7 +69,9 @@ def news( ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = get_data_records(parse_json(response), exclude_keys=["s"]) + data = get_data_records( + _to_fields(output_model, parse_json(response)), exclude_keys=["s"] + ) with model_errors(response): return [output_model(**row) for row in data] diff --git a/src/marketdata/resources/stocks/prices.py b/src/marketdata/resources/stocks/prices.py index 2674dbc..615410b 100644 --- a/src/marketdata/resources/stocks/prices.py +++ b/src/marketdata/resources/stocks/prices.py @@ -8,6 +8,7 @@ ) from marketdata.input_types.stocks import StocksPricesInput from marketdata.output_handlers import get_dataframe_output_handler +from marketdata.output_types.columns import _to_fields from marketdata.output_types.stocks_prices import StockPrice, StockPricesHumanReadable from marketdata.params import universal_params from marketdata.resources.base import model_errors, no_data_result @@ -67,9 +68,11 @@ def prices( ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = get_data_records(parse_json(response, exact=True)) + data = get_data_records( + _to_fields(output_model, parse_json(response, exact=True)) + ) with model_errors(response): - return [output_model.from_dict(row) for row in data] + return [output_model(**row) for row in data] elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/stocks/quotes.py b/src/marketdata/resources/stocks/quotes.py index 9adb775..026560d 100644 --- a/src/marketdata/resources/stocks/quotes.py +++ b/src/marketdata/resources/stocks/quotes.py @@ -5,6 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.stocks import StocksQuotesInput from marketdata.output_handlers import get_dataframe_output_handler +from marketdata.output_types.columns import _to_fields from marketdata.output_types.stocks_quotes import StockQuote, StockQuotesHumanReadable from marketdata.params import universal_params from marketdata.resources.base import BaseResource, model_errors, no_data_result @@ -64,9 +65,12 @@ def quotes( ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = get_data_records(parse_json(response, exact=True), exclude_keys=["s"]) + data = get_data_records( + _to_fields(output_model, parse_json(response, exact=True)), + exclude_keys=["s"], + ) with model_errors(response): - return [output_model.from_dict(row) for row in data] + return [output_model(**row) for row in data] elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/utilities/base.py b/src/marketdata/resources/utilities/base.py index 8237e6e..f054879 100644 --- a/src/marketdata/resources/utilities/base.py +++ b/src/marketdata/resources/utilities/base.py @@ -14,6 +14,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.output_handlers import get_dataframe_output_handler +from marketdata.output_types.columns import _to_fields from marketdata.resources.base import BaseResource, model_errors, no_data_result from marketdata.utils import dict_to_csv, get_data_records, is_no_data, parse_json @@ -69,7 +70,7 @@ def render( elif output_format == OutputFormat.INTERNAL: if as_records: - rows = get_data_records(data, exclude_keys=["s"]) + rows = get_data_records(_to_fields(output_model, data), exclude_keys=["s"]) with model_errors(response): return [output_model(**row) for row in rows] with model_errors(response): diff --git a/src/marketdata/utils.py b/src/marketdata/utils.py index 180f209..85cbd6d 100644 --- a/src/marketdata/utils.py +++ b/src/marketdata/utils.py @@ -362,10 +362,11 @@ def merge_csv_responses( Args: responses: The answers, in the order their rows go in the file. - known_columns: The resource's column names. A header name must match - one of them through ``column_key``, which also compares the - headers of two answers, so a different spelling of the same - column still merges. + known_columns: The resource's column names. A header must carry at + least one of them, matched through ``column_key``, which also + compares the headers of two answers, so a different spelling of + the same column still merges. A column of the header that matches + none of them is kept. with_header: Whether the bodies start with a header row. Without one, only the row width is checked, against the first row seen. @@ -376,10 +377,10 @@ def merge_csv_responses( Raises: ParseError: Naming the offending response, for a body the ``csv`` - module cannot read, a body with no header row, a header with a - column the resource does not have or with other columns, or in - another order, than the first answer's, or a row not as wide as - the header. + module cannot read, a body with no header row, a header with none + of the resource's columns or with other columns, or in another + order, than the first answer's, or a row not as wide as the + header. """ known = {column_key(name) for name in known_columns} header: list[str] | None = None @@ -395,9 +396,8 @@ def merge_csv_responses( incoming = next(rows, None) if incoming is None: raise parse_error(response, "no header row") - unknown = [name for name in incoming if column_key(name) not in known] - if unknown: - raise parse_error(response, f"unknown columns {unknown!r}") + if not any(column_key(name) in known for name in incoming): + raise parse_error(response, f"unknown columns {incoming!r}") incoming_key = [column_key(name) for name in incoming] if header is None: header, header_key, width = incoming, incoming_key, len(incoming) diff --git a/src/tests/test_money_decimal.py b/src/tests/test_money_decimal.py index f578024..10f77c4 100644 --- a/src/tests/test_money_decimal.py +++ b/src/tests/test_money_decimal.py @@ -667,27 +667,15 @@ def test_a_value_the_model_cannot_hold_is_a_parse_error_not_a_builtin( assert body["mid"] == [decoded] -@pytest.mark.parametrize( - "name, change, refusal", - [ - ("options.lookup", "extra", "got an unexpected keyword argument 'extra'"), - ("markets.status", "extra", "got an unexpected keyword argument 'extra'"), - ("utilities.user", "missing", "missing 1 required positional argument"), - ], -) -def test_a_body_whose_keys_the_model_does_not_take_is_a_parse_error( - respx_mock, client, name, change, refusal +@pytest.mark.parametrize("name", ["options.lookup", "markets.status", "utilities.user"]) +def test_a_body_missing_a_column_the_model_needs_is_a_parse_error( + respx_mock, client, name ): - """A key the model does not have, or one it lacks, used to escape as a - bare `TypeError` from the model's constructor (#50 review). What the SDK - should do with a column the API adds is #111; this pins only that the - refusal is an SDK exception.""" + """A column the model needs and the body lacks is refused with an SDK + exception, not a bare `TypeError` from the model's constructor.""" case = OTHER_CASES[name] data = _load_fixture(case.fixture) - if change == "extra": - data["extra"] = data[next(key for key in data if key != "s")] - else: - del data[next(key for key in data if key != "s")] + del data[next(key for key in data if key != "s")] _respond(respx_mock, case, json.dumps(data).encode()) with pytest.raises(ParseError) as failure: @@ -695,7 +683,108 @@ def test_a_body_whose_keys_the_model_does_not_take_is_a_parse_error( message = failure.value.message assert message.startswith("Response body is not a valid answer of this resource (") - assert refusal in message + assert "missing 1 required positional argument" in message + + +# Every INTERNAL resource whose answer is built into a model through its +# columns: `utilities.headers` keeps every key as a header, and +# `utilities.user` reads only the keys it knows. +MODEL_CASES = { + name: case + for name, case in {**MONEY_CASES, **OTHER_CASES}.items() + if name not in ("utilities.headers", "utilities.user") +} +# They merge only the columns their model declares, before any model is built. +FAN_OUTS = ( + "stocks.candles", + "stocks.candles human", + "options.quotes", + "options.quotes human", +) + + +def _undeclared_columns_logged(caplog) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if "does not declare" in record.getMessage() + ] + + +@pytest.mark.parametrize("name", MODEL_CASES) +def test_a_column_no_model_declares_is_left_out_of_the_internal_result( + respx_mock, client, caplog, name +): + """A column the API adds before the model declares it does not break the + call: the result is the one the answer builds without it, and the column + is named once at DEBUG, however many rows the answer has.""" + case = MODEL_CASES[name] + data = _load_fixture(case.fixture) + _respond(respx_mock, case, json.dumps(data).encode()) + with caplog.at_level("DEBUG", logger="marketdata.logger"): + expected = case.call(client, output_format=OutputFormat.INTERNAL) + assert _undeclared_columns_logged(caplog) == [] + + data["brandNew"] = data[next(key for key in data if key != "s")] + _respond(respx_mock, case, json.dumps(data).encode()) + caplog.clear() + with caplog.at_level("DEBUG", logger="marketdata.logger"): + result = case.call(client, output_format=OutputFormat.INTERNAL) + + assert result == expected + model = type(result[0] if isinstance(result, list) else result).__name__ + logged = [ + f"The API sent the columns ['brandNew'], which {model} does not declare;" + " they are left out" + ] + assert _undeclared_columns_logged(caplog) == ([] if name in FAN_OUTS else logged) + + +@pytest.mark.parametrize("name", MODEL_CASES) +def test_an_object_with_none_of_the_model_columns_is_a_parse_error( + respx_mock, client, name +): + """An object that carries none of the model's columns, like a proxy's JSON + error page, is not an answer of this resource, so no row is built from + what is left of it.""" + case = MODEL_CASES[name] + _respond(respx_mock, case, json.dumps({"error": "upstream timeout"}).encode()) + + with pytest.raises(ParseError) as failure: + case.call(client, output_format=OutputFormat.INTERNAL) + + assert failure.value.message.startswith( + "Response body is not a valid answer of this resource (" + ) + + +@pytest.mark.parametrize("body", [b"[1]", b"null"]) +@pytest.mark.parametrize( + "name", + [ + "options.chain", + "options.chain human", + "options.expirations", + "options.expirations human", + "options.lookup", + "stocks.earnings", + "stocks.earnings human", + ], +) +def test_a_body_that_is_not_an_object_is_a_parse_error_on_single_object_models( + respx_mock, client, name, body +): + """A single-object model refuses a body that is not a JSON object with an + SDK exception, under the API's names and the human-readable ones.""" + case = MODEL_CASES[name] + _respond(respx_mock, case, body) + + with pytest.raises(ParseError) as failure: + case.call(client, output_format=OutputFormat.INTERNAL) + + assert failure.value.message.startswith( + "Response body is not a valid answer of this resource (" + ) @pytest.mark.parametrize("name", OTHER_DATE_KEYS.keys()) diff --git a/src/tests/test_options_quotes.py b/src/tests/test_options_quotes.py index 9704e1a..e39afb5 100644 --- a/src/tests/test_options_quotes.py +++ b/src/tests/test_options_quotes.py @@ -879,6 +879,23 @@ def test_options_quotes_csv_merges_every_symbol_under_the_api_header( ) +def test_options_quotes_csv_keeps_a_column_the_api_adds(respx_mock, client, tmp_path): + """A column the model does not declare is written with the others for + every symbol, as the CSV of a single-request resource carries it.""" + respx_mock.get(CALL_URL).respond(text=f"{CSV_HEADER},brandNew\n{CALL_ROW},x\n") + respx_mock.get(PUT_URL).respond(text=f"{CSV_HEADER},brandNew\n{PUT_ROW},y\n") + + output = client.options.quotes( + symbols=["AAPL271217C00255000", "AAPL271217P00255000"], + output_format=OutputFormat.CSV, + filename=tmp_path / "test.csv", + ) + + assert pathlib.Path(output).read_bytes() == ( + f"{CSV_HEADER},brandNew\r\n{CALL_ROW},x\r\n{PUT_ROW},y\r\n".encode() + ) + + def test_options_quotes_csv_keeps_the_requested_columns(respx_mock, client, tmp_path): """Issue #86: under `columns=` the API answers with the requested columns only; the merge used to drop every row because the header did not match diff --git a/src/tests/test_stocks_candles.py b/src/tests/test_stocks_candles.py index af4ff9a..a13a6a3 100644 --- a/src/tests/test_stocks_candles.py +++ b/src/tests/test_stocks_candles.py @@ -516,6 +516,27 @@ def test_stocks_candles_csv_merges_every_chunk_under_the_api_header( assert pathlib.Path(output).read_bytes() == b"t,c\r\n1,2\r\n3,4\r\n" +def test_stocks_candles_csv_keeps_a_column_the_api_adds(respx_mock, client, tmp_path): + """A column the model does not declare is written with the others from + every chunk, as the CSV of a single-request resource carries it.""" + respx_mock.get(HOURLY_URL).mock( + side_effect=by_chunk( + dict(text="t,c,brandNew\r\n1,2,x\r\n"), + dict(text="t,c,brandNew\r\n3,4,y\r\n"), + ) + ) + + output = client.stocks.candles( + symbol="AAPL", + resolution="H", + output_format=OutputFormat.CSV, + filename=tmp_path / "test.csv", + **TWO_CHUNKS, + ) + + assert pathlib.Path(output).read_bytes() == b"t,c,brandNew\r\n1,2,x\r\n3,4,y\r\n" + + def test_stocks_candles_csv_undecodable_chunk_body_is_a_parse_error( respx_mock, client, tmp_path ): diff --git a/src/tests/test_utils.py b/src/tests/test_utils.py index 85c9c1e..931c72a 100644 --- a/src/tests/test_utils.py +++ b/src/tests/test_utils.py @@ -163,6 +163,17 @@ def test_merge_csv_responses_accepts_human_readable_names_for_the_model_fields() assert result == "Expiration Date,Strike\r\n1,2\r\n" +def test_merge_csv_responses_keeps_a_column_the_resource_does_not_have(): + responses = [ + _csv_response("t,c,brandNew\n1,2,x\n"), + _csv_response("t,c,brandNew\n3,4,y\n"), + ] + + result = merge_csv_responses(responses, COLUMNS) + + assert result == "t,c,brandNew\r\n1,2,x\r\n3,4,y\r\n" + + @pytest.mark.parametrize( ("bodies", "reason"), [ From 6a6b3db375ebaea76152c5af06b299e2e9d318f4 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 09:10:46 -0300 Subject: [PATCH 02/13] test(stocks): pin StockPrice.from_dict and its human-readable twin directly stocks.prices builds its rows with the constructor now, so neither from_dict ran in the suite and Codecov reported both lines uncovered. Each test builds one row that carries a column the model does not declare. --- src/tests/test_stocks_prices.py | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/tests/test_stocks_prices.py b/src/tests/test_stocks_prices.py index 787c09b..15691ae 100644 --- a/src/tests/test_stocks_prices.py +++ b/src/tests/test_stocks_prices.py @@ -49,6 +49,54 @@ def test_stock_prices_human_readable_str(): assert isinstance(str(instance), str) +def test_stock_price_from_dict_leaves_out_an_undeclared_column(): + """StockPrice.from_dict builds the price from one row keyed by the API's + names and leaves out a column the model does not declare.""" + instance = StockPrice.from_dict( + { + "s": "ok", + "symbol": "AAPL", + "mid": 280.02, + "change": -0.68, + "changepct": -0.0024, + "updated": 1765478200, + "brandNew": 1, + } + ) + + assert instance == StockPrice( + s="ok", + symbol="AAPL", + mid=280.02, + change=-0.68, + changepct=-0.0024, + updated=1765478200, + ) + + +def test_stock_prices_human_readable_from_dict_leaves_out_an_undeclared_column(): + """StockPricesHumanReadable.from_dict builds the price from one row keyed by + the human-readable names and leaves out a column the model does not declare.""" + instance = StockPricesHumanReadable.from_dict( + { + "Symbol": "AAPL", + "Mid": 280.02, + "Change $": -0.68, + "Change %": -0.0024, + "Date": 1765478200, + "brandNew": 1, + } + ) + + assert instance == StockPricesHumanReadable( + Symbol="AAPL", + Mid=280.02, + Change_Price=-0.68, + Change_Percent=-0.0024, + Date=1765478200, + ) + + def test_get_stocks_prices_response_200_internal(load_json, respx_mock, client): mock_data = load_json("stocks_prices_response_200") From b8035b2907b9d01216b31c32285247a45736cc78 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 11:20:02 -0300 Subject: [PATCH 03/13] refactor(models): keep the column mapping's logger private Every module-level name without a leading underscore is part of the SDK's public surface, so `logger` added `marketdata.output_types.columns.logger`. `_logger` is the same `marketdata.logger` logger under a private name. --- src/marketdata/output_types/columns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/marketdata/output_types/columns.py b/src/marketdata/output_types/columns.py index 5cff0dc..d039964 100644 --- a/src/marketdata/output_types/columns.py +++ b/src/marketdata/output_types/columns.py @@ -5,7 +5,7 @@ from marketdata.utils import column_key # By name, not `get_logger()`: importing the package must not attach a handler. -logger = logging.getLogger("marketdata.logger") +_logger = logging.getLogger("marketdata.logger") def _column_names(output_model: type) -> dict[str, str]: @@ -77,7 +77,7 @@ def _to_fields(output_model: type, data: Any) -> Any: by_key.update({column: name for name, column in columns.items()}) undeclared = [key for key in data if key not in by_key and key != "s"] if undeclared: - logger.debug( + _logger.debug( f"The API sent the columns {undeclared!r}, which" f" {output_model.__name__} does not declare; they are left out" ) From 9f65a39e3b11ea67be95b38baf1ce5fc749648c6 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 12:32:01 -0300 Subject: [PATCH 04/13] fix(resources): refuse a non-object body the same way on every decoded format options.chain, options.expirations, options.lookup and stocks.earnings raised ParseError for a body that is not a JSON object only on INTERNAL: JSON returned the body and DATAFRAME built a frame from it, an empty one for null. Their three decoded formats now decode through one check and raise the same ParseError. The CSV file stays the API's text as it came, since that format decodes nothing. --- CHANGELOG.md | 2 +- src/marketdata/resources/base.py | 21 ++++++++++++ src/marketdata/resources/options/chain.py | 15 ++++++--- .../resources/options/expirations.py | 15 ++++++--- src/marketdata/resources/options/lookup.py | 15 ++++++--- src/marketdata/resources/stocks/earnings.py | 15 ++++++--- src/tests/test_money_decimal.py | 32 +++++++++++++------ 7 files changed, 85 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec60c31..82b4fd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Money in the `OutputFormat.INTERNAL` models is `decimal.Decimal` instead of `float`, built from the digits in the response body, which those resources now decode with `parse_float=Decimal` on the INTERNAL path: `ask`, `bid`, `mid`, `last`, `change` of `StockQuote`, `mid` and `change` of `StockPrice`, `o`, `h`, `l`, `c` of `StockCandle` and `FundsCandle`, the three EPS lists of `StockEarnings`, `strike`, `bid`, `mid`, `ask`, `last`, `intrinsicValue`, `extrinsicValue`, `underlyingPrice` of `OptionsChain` and `OptionsQuotes`, and the same fields of every human-readable twin. Greeks, IV and percentages stay `float`, sizes and counts `int`; DataFrames (pandas and polars), JSON and CSV output keep the plain float parse. Migration: arithmetic mixing `Decimal` and `float` raises `TypeError`, and a comparison with a float literal compares against its binary value (`Decimal("65.1") == 65.1` is `False`), so use `Decimal` literals or `int`; `float(value)` gives a float back, and `OutputFormat.DATAFRAME` is the float path for analysis (a pandas DataFrame built from models gets `object` columns, and `json.dumps(..., default=str)` writes the money as strings). A model built by hand converts its money arguments: a float or a numpy scalar through its shortest repr (`65.1` is `Decimal("65.1")`), a NaN number, which is how pandas writes a missing value, to `None`; a value that is not a number, a string that reads as NaN, or an infinity raises and names the field, while an array (anything that is not a string and has `__array__` and `__len__`: a numpy array, a pandas or polars Series, a pandas Index) is left as it came, and a `dict`, a `set` or a generator raises. On the API path the same refusal is a `ParseError` naming the field and the body, not a bare `TypeError` (#50) - Dates on `OutputFormat.INTERNAL` are read with the API's own rule and always come back as US/Eastern datetimes. A number from 10000 up to 200000 is a spreadsheet serial of US/Eastern wall-clock time, from 200000 Unix seconds, from `1e10` milliseconds and from `1e13` nanoseconds. A number under 10000 is a relative range for the API, not a date, and raises `ParseError`. The SDK used a boundary of its own at 60000, so from April 2064 a serial would have read as 1970. A `timestamp` date (`2026-09-21`) and a spreadsheet serial are now midnight and wall-clock time in US/Eastern, where they came back as naive datetimes, and a serial is rounded to the second. The intraday `from_date` and `to_date` of `stocks.candles()` are read only when they are ISO dates or numbers the API reads as dates (a spreadsheet serial or a Unix time), which the SDK splits into one request per year: a relative range or a keyword goes to the API as it is, in one request, where `"60"` was sent as 1900-02-28 and `"yesterday"` was refused. `UserRateLimits.reset_time` takes a datetime or Unix seconds, and refuses anything else (#116). Migration: compare INTERNAL dates against aware datetimes (`pytz.timezone("US/Eastern").localize(...)`), not naive ones - A response body carrying a `NaN`, `Infinity` or `-Infinity` literal raises `ParseError` on every output format that decodes it, where `INTERNAL`, `JSON` and `DATAFRAME` used to return it as `nan` or `inf`. Those literals are not JSON, and the renderer behind the API's data endpoints refuses them, answering `500`, so one in their bodies was written by something in between (`/status/` and `/headers/` are served elsewhere). `CSV` output writes the API's body as it came, since it reads no number, except for `client.utilities`, which builds its CSV from the decoded body and refuses it too. A number past what a float holds is not refused: `JSON` and `DATAFRAME` read `1e400` as `inf` while `INTERNAL` keeps it exact, and `INTERNAL` refuses one past what a `Decimal` holds. That renderer does not write such a number, and refusing it on the float path would slow every decode. Migration: code that read `nan` or `inf` from such a body catches `ParseError` (or `BaseMarketdataException`) instead; no setting brings the literals back (#50) -- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones; elsewhere it raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) +- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`); their CSV file is the API's text as it came, as on every resource. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) - `options.chain(delta=...)` checks a number before it sends it, the way it checks `strike`, and keeps it as the text it sends. A `bool`, a NaN, an infinity, a number a float cannot hold and a delta outside -1 to 1 now raise `ValidationError` before any request, where they used to be sent: `delta=nan` came back as the whole chain, since the API reads it as no filter, `delta=True` was sent as `1.0`, and `delta=30` was sent as `30.0`. `OptionsChainInput.delta` holds that text (`"0.5"`) instead of a `float` (#101). Migration: pass a finite number from -1 to 1, the way the public docs write a delta (`0.3`), a `DeltaFilter`, or `delta=None` for no filter; code that reads `OptionsChainInput(...).delta` gets the text sent to the API, and `float()` turns a plain number back. - Every date field of the `OutputFormat.INTERNAL` models admits `None` in its annotation: `datetime.datetime | None` for one date, `list[datetime.datetime | None]` for a column of them, and `datetime.date | None` for `MarketStatus.date`, as `OptionsExpirations.updated` already did. A `null` date in an answer reads as `None` in its own row (see Fixed), and the annotations now say so. Migration: a type checker now asks for a `None` check before a `datetime` method is called on a date field (`if quote.updated is not None: quote.updated.date()`). At run time nothing that used to return changes: an answer with a `null` date used to raise `ParseError`, or on `OptionsQuotes` shift the column (#106) diff --git a/src/marketdata/resources/base.py b/src/marketdata/resources/base.py index a23830b..c061646 100644 --- a/src/marketdata/resources/base.py +++ b/src/marketdata/resources/base.py @@ -52,6 +52,27 @@ def model_columns(output_model: type, requested: list[str] | None = None) -> lis return _model_columns(output_model, requested) +def _parse_json_object(response: Response, *, exact: bool = False) -> dict[str, Any]: + """Decode a response body that must be a JSON object. + + Args: + response: The answer to decode. + exact: Decode each number with a fraction as a ``Decimal``, as + ``parse_json`` does. + + Returns: + The decoded object. + + Raises: + ParseError: If the body is not valid JSON, holds a number that is not + finite, or is not a JSON object. + """ + data = parse_json(response, exact=exact) + if not isinstance(data, dict): + raise parse_error(response, "not a JSON object") + return data + + @contextmanager def model_errors(response: Response) -> Iterator[None]: """Turn a model refusing a value into a ``ParseError``. diff --git a/src/marketdata/resources/options/chain.py b/src/marketdata/resources/options/chain.py index 9cdb90e..4b6249b 100644 --- a/src/marketdata/resources/options/chain.py +++ b/src/marketdata/resources/options/chain.py @@ -11,8 +11,13 @@ OptionsChainHumanReadable, ) from marketdata.params import universal_params -from marketdata.resources.base import BaseResource, model_errors, no_data_result -from marketdata.utils import encode_path_segment, is_no_data, parse_json +from marketdata.resources.base import ( + BaseResource, + _parse_json_object, + model_errors, + no_data_result, +) +from marketdata.utils import encode_path_segment, is_no_data @api_error_handler(service="/v1/options/chain/") @@ -61,19 +66,19 @@ def chain( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_object(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["optionSymbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = parse_json(response, exact=True) + data = _parse_json_object(response, exact=True) with model_errors(response): return output_model(**_to_fields(output_model, data)) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_object(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/marketdata/resources/options/expirations.py b/src/marketdata/resources/options/expirations.py index 268419c..322daec 100644 --- a/src/marketdata/resources/options/expirations.py +++ b/src/marketdata/resources/options/expirations.py @@ -11,8 +11,13 @@ OptionsExpirationsHumanReadable, ) from marketdata.params import universal_params -from marketdata.resources.base import BaseResource, model_errors, no_data_result -from marketdata.utils import encode_path_segment, is_no_data, parse_json +from marketdata.resources.base import ( + BaseResource, + _parse_json_object, + model_errors, + no_data_result, +) +from marketdata.utils import encode_path_segment, is_no_data @api_error_handler(service="/v1/options/expirations/") @@ -73,19 +78,19 @@ def expirations( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_object(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=index_columns ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = parse_json(response) + data = _parse_json_object(response) with model_errors(response): return output_model(**_to_fields(output_model, data)) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_object(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/marketdata/resources/options/lookup.py b/src/marketdata/resources/options/lookup.py index 6b7bc10..9ccde9f 100644 --- a/src/marketdata/resources/options/lookup.py +++ b/src/marketdata/resources/options/lookup.py @@ -11,8 +11,13 @@ OptionsLookupHumanReadable, ) from marketdata.params import universal_params -from marketdata.resources.base import BaseResource, model_errors, no_data_result -from marketdata.utils import encode_path, is_no_data, parse_json +from marketdata.resources.base import ( + BaseResource, + _parse_json_object, + model_errors, + no_data_result, +) +from marketdata.utils import encode_path, is_no_data @api_error_handler(service="/v1/options/lookup/") @@ -65,19 +70,19 @@ def lookup( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_object(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["optionSymbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = parse_json(response) + data = _parse_json_object(response) with model_errors(response): return output_model(**_to_fields(output_model, data)) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_object(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/marketdata/resources/stocks/earnings.py b/src/marketdata/resources/stocks/earnings.py index ef3f089..15bde64 100644 --- a/src/marketdata/resources/stocks/earnings.py +++ b/src/marketdata/resources/stocks/earnings.py @@ -10,8 +10,13 @@ StockEarningsHumanReadable, ) from marketdata.params import universal_params -from marketdata.resources.base import BaseResource, model_errors, no_data_result -from marketdata.utils import encode_path_segment, is_no_data, parse_json +from marketdata.resources.base import ( + BaseResource, + _parse_json_object, + model_errors, + no_data_result, +) +from marketdata.utils import encode_path_segment, is_no_data @api_error_handler(service="/v1/stocks/earnings/") @@ -60,19 +65,19 @@ def earnings( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_object(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["symbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = parse_json(response, exact=True) + data = _parse_json_object(response, exact=True) with model_errors(response): return output_model.from_dict(data) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_object(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/tests/test_money_decimal.py b/src/tests/test_money_decimal.py index 10f77c4..15ba0f2 100644 --- a/src/tests/test_money_decimal.py +++ b/src/tests/test_money_decimal.py @@ -771,21 +771,35 @@ def test_an_object_with_none_of_the_model_columns_is_a_parse_error( "stocks.earnings human", ], ) -def test_a_body_that_is_not_an_object_is_a_parse_error_on_single_object_models( - respx_mock, client, name, body +def test_a_body_that_is_not_an_object_is_one_parse_error_on_every_decoded_format( + respx_mock, client, tmp_path, name, body ): - """A single-object model refuses a body that is not a JSON object with an - SDK exception, under the API's names and the human-readable ones.""" + """A single-object resource refuses a body that is not a JSON object with + the same ``ParseError`` on every format that decodes it, under the API's + names and the human-readable ones. The CSV file is the API's text as it + came, since that format decodes nothing.""" case = MODEL_CASES[name] _respond(respx_mock, case, body) - with pytest.raises(ParseError) as failure: - case.call(client, output_format=OutputFormat.INTERNAL) - - assert failure.value.message.startswith( - "Response body is not a valid answer of this resource (" + messages = set() + for output_format in ( + OutputFormat.INTERNAL, + OutputFormat.JSON, + OutputFormat.DATAFRAME, + ): + with pytest.raises(ParseError) as failure: + case.call(client, output_format=output_format) + messages.add(failure.value.message) + path = case.call( + client, output_format=OutputFormat.CSV, filename=tmp_path / "answer.csv" ) + assert messages == { + "Response body is not a valid answer of this resource (not a JSON object):" + f" {body.decode()!r}" + } + assert pathlib.Path(path).read_bytes() == body + @pytest.mark.parametrize("name", OTHER_DATE_KEYS.keys()) def test_a_date_the_model_cannot_read_is_a_parse_error_with_no_money_involved( From d48305572540e74bd1d5f5f15222e15ae764ae1a Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 13:24:13 -0300 Subject: [PATCH 05/13] feat(models)!: keep the columns a model does not declare for get_extra An INTERNAL answer that carries a column its model does not declare keeps it: marketdata.get_extra(model) returns it as the API sent it, the whole column on a single-object model and each row's own value on a list result, and the DEBUG line says where it went. The from_dict of StockEarnings, StockPrice, StockQuote and their human-readable twins keep such a key the same way. The fan-outs merge only the columns their model declares, so their INTERNAL, JSON and DATAFRAME output still leave it out. The CSV output of options.chain, options.expirations, options.lookup and stocks.earnings refuses a header row with none of the resource's columns, the rule the fan-out merge applies, so a body that is not an answer raises ParseError on every format. BREAKING CHANGE: an INTERNAL call whose answer carries a column the model does not declare returns the model, where it raised TypeError, and the from_dict methods accept such a key, where they raised TypeError. options.chain, options.expirations, options.lookup and stocks.earnings raise ParseError for a body that is not a JSON object on JSON and DATAFRAME, where they returned it, and for a CSV header row with none of their columns, where they wrote the file. The CSV of stocks.candles and options.quotes keeps a column the API adds, where it raised ParseError. --- CHANGELOG.md | 5 +- src/marketdata/__init__.py | 2 + src/marketdata/output_types/columns.py | 105 ++++++++-- .../output_types/stocks_earnings.py | 26 ++- src/marketdata/output_types/stocks_prices.py | 26 ++- src/marketdata/output_types/stocks_quotes.py | 26 ++- src/marketdata/resources/base.py | 29 +++ src/marketdata/resources/funds/candles.py | 7 +- src/marketdata/resources/markets/status.py | 7 +- src/marketdata/resources/options/chain.py | 11 +- .../resources/options/expirations.py | 11 +- src/marketdata/resources/options/lookup.py | 11 +- src/marketdata/resources/stocks/earnings.py | 6 + src/marketdata/resources/stocks/news.py | 9 +- src/marketdata/resources/stocks/prices.py | 9 +- src/marketdata/resources/stocks/quotes.py | 10 +- src/marketdata/resources/utilities/base.py | 7 +- src/tests/test_money_decimal.py | 180 ++++++++++++++++-- src/tests/test_options_chain.py | 7 +- src/tests/test_options_expirations.py | 7 +- src/tests/test_options_lookup.py | 4 +- src/tests/test_stocks_earnings.py | 7 +- src/tests/test_stocks_prices.py | 12 +- 23 files changed, 435 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82b4fd7..01a2b81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Money in the `OutputFormat.INTERNAL` models is `decimal.Decimal` instead of `float`, built from the digits in the response body, which those resources now decode with `parse_float=Decimal` on the INTERNAL path: `ask`, `bid`, `mid`, `last`, `change` of `StockQuote`, `mid` and `change` of `StockPrice`, `o`, `h`, `l`, `c` of `StockCandle` and `FundsCandle`, the three EPS lists of `StockEarnings`, `strike`, `bid`, `mid`, `ask`, `last`, `intrinsicValue`, `extrinsicValue`, `underlyingPrice` of `OptionsChain` and `OptionsQuotes`, and the same fields of every human-readable twin. Greeks, IV and percentages stay `float`, sizes and counts `int`; DataFrames (pandas and polars), JSON and CSV output keep the plain float parse. Migration: arithmetic mixing `Decimal` and `float` raises `TypeError`, and a comparison with a float literal compares against its binary value (`Decimal("65.1") == 65.1` is `False`), so use `Decimal` literals or `int`; `float(value)` gives a float back, and `OutputFormat.DATAFRAME` is the float path for analysis (a pandas DataFrame built from models gets `object` columns, and `json.dumps(..., default=str)` writes the money as strings). A model built by hand converts its money arguments: a float or a numpy scalar through its shortest repr (`65.1` is `Decimal("65.1")`), a NaN number, which is how pandas writes a missing value, to `None`; a value that is not a number, a string that reads as NaN, or an infinity raises and names the field, while an array (anything that is not a string and has `__array__` and `__len__`: a numpy array, a pandas or polars Series, a pandas Index) is left as it came, and a `dict`, a `set` or a generator raises. On the API path the same refusal is a `ParseError` naming the field and the body, not a bare `TypeError` (#50) - Dates on `OutputFormat.INTERNAL` are read with the API's own rule and always come back as US/Eastern datetimes. A number from 10000 up to 200000 is a spreadsheet serial of US/Eastern wall-clock time, from 200000 Unix seconds, from `1e10` milliseconds and from `1e13` nanoseconds. A number under 10000 is a relative range for the API, not a date, and raises `ParseError`. The SDK used a boundary of its own at 60000, so from April 2064 a serial would have read as 1970. A `timestamp` date (`2026-09-21`) and a spreadsheet serial are now midnight and wall-clock time in US/Eastern, where they came back as naive datetimes, and a serial is rounded to the second. The intraday `from_date` and `to_date` of `stocks.candles()` are read only when they are ISO dates or numbers the API reads as dates (a spreadsheet serial or a Unix time), which the SDK splits into one request per year: a relative range or a keyword goes to the API as it is, in one request, where `"60"` was sent as 1900-02-28 and `"yesterday"` was refused. `UserRateLimits.reset_time` takes a datetime or Unix seconds, and refuses anything else (#116). Migration: compare INTERNAL dates against aware datetimes (`pytz.timezone("US/Eastern").localize(...)`), not naive ones - A response body carrying a `NaN`, `Infinity` or `-Infinity` literal raises `ParseError` on every output format that decodes it, where `INTERNAL`, `JSON` and `DATAFRAME` used to return it as `nan` or `inf`. Those literals are not JSON, and the renderer behind the API's data endpoints refuses them, answering `500`, so one in their bodies was written by something in between (`/status/` and `/headers/` are served elsewhere). `CSV` output writes the API's body as it came, since it reads no number, except for `client.utilities`, which builds its CSV from the decoded body and refuses it too. A number past what a float holds is not refused: `JSON` and `DATAFRAME` read `1e400` as `inf` while `INTERNAL` keeps it exact, and `INTERNAL` refuses one past what a `Decimal` holds. That renderer does not write such a number, and refusing it on the float path would slow every decode. Migration: code that read `nan` or `inf` from such a body catches `ParseError` (or `BaseMarketdataException`) instead; no setting brings the literals back (#50) -- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`); their CSV file is the API's text as it came, as on every resource. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) +- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`), and their CSV output raises `ParseError` too, since it now refuses a header row with none of the resource's columns, where it wrote the body to the file. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) - `options.chain(delta=...)` checks a number before it sends it, the way it checks `strike`, and keeps it as the text it sends. A `bool`, a NaN, an infinity, a number a float cannot hold and a delta outside -1 to 1 now raise `ValidationError` before any request, where they used to be sent: `delta=nan` came back as the whole chain, since the API reads it as no filter, `delta=True` was sent as `1.0`, and `delta=30` was sent as `30.0`. `OptionsChainInput.delta` holds that text (`"0.5"`) instead of a `float` (#101). Migration: pass a finite number from -1 to 1, the way the public docs write a delta (`0.3`), a `DeltaFilter`, or `delta=None` for no filter; code that reads `OptionsChainInput(...).delta` gets the text sent to the API, and `float()` turns a plain number back. - Every date field of the `OutputFormat.INTERNAL` models admits `None` in its annotation: `datetime.datetime | None` for one date, `list[datetime.datetime | None]` for a column of them, and `datetime.date | None` for `MarketStatus.date`, as `OptionsExpirations.updated` already did. A `null` date in an answer reads as `None` in its own row (see Fixed), and the annotations now say so. Migration: a type checker now asks for a `None` check before a `datetime` method is called on a date field (`if quote.updated is not None: quote.updated.date()`). At run time nothing that used to return changes: an answer with a `null` date used to raise `ParseError`, or on `OptionsQuotes` shift the column (#106) +- A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model is built without it, where the call raised a bare `TypeError`, and `marketdata.get_extra(model)` returns it as the API sent it: the whole column on a single-object model, the row's own value on each row of a list result. The column is named once per answer at DEBUG on `marketdata.logger`. `StockEarnings.from_dict()`, `StockPrice.from_dict()`, `StockQuote.from_dict()` and their human-readable twins keep such a key for `get_extra()` the same way, where they raised `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` still leave such a column out of their INTERNAL, DataFrame and JSON output when they merge, with nothing for `get_extra()`; their CSV output keeps it, where it raised `ParseError`, because `merge_csv_responses()` now refuses only a header with none of the resource's columns. Migration: code that caught the `TypeError` or `ParseError` an added column raised now gets the result; read the column with `get_extra()` until the model declares it (#111) ### Changed @@ -40,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `marketdata.get_extra(model)` reads the columns an `OutputFormat.INTERNAL` answer carried that its model does not declare, keyed as the API sent them (#111) - `StrikeFilter` builds the `strike` filter of `options.chain` from its shapes instead of from a hand-written string: `exact`, `any_of`, `between`, `at_least`, `at_most`, `above`, `below`, and `expression` for anything the others do not name, the way sdk-go and sdk-csharp give them. `DeltaFilter` does the same for `delta`, which the API reads with the same grammar, with `nearest` in place of `exact`: the API answers a single delta with the contract nearest to it. Every `DeltaFilter` constructor but `expression` raises `ValueError` for a delta outside -1 to 1. Both parameters also take a plain `int`, `float` or `Decimal` now, which `strike` refused before while the public docs typed it `float`, so a strike read from an INTERNAL chain can be passed straight back. A negative **number** is refused rather than sent: the API reads a number by its absolute value, so `strike=-250` used to answer with the strikes at `250` and no error. A string is still passed through as written, since it is the expression form and the API is what reads it. A negative delta is kept, since there the absolute value is the documented filter and both sides come back either way. A number travels as plain digits, never in exponent form: the API reads a query of `delta=0.00005`, and answers `delta=5e-05` with a `400`, since it splits that text at its minus. A number a float cannot hold is refused, since the API reads each one with `float()` and would take it for an infinity or for 0 (#101) - The API's IP headers, for accounts that restrict access by IP (#44). A 403 that blocks an address raises `ForbiddenError` with `authorized_ip`, the address the account is authorized for, taken from `X-API-Authorized-IP` and repeated in the message, so a caller who prints the error knows what to allow. `marketdata.get_meta(result).detected_ip` is the address the call came from, taken from `X-API-Detected-IP` on every answer the API serves. Both read `None` when the API sent no header: an account allowed to call from several addresses, a staff token, the Sheets add-on, or an address it could not resolve. The legacy `X-API-BLOCKED-IP` is not read: the API is removing it (MarketData-App/api#202) - `client.utilities` resource with `status()`, `headers()` and `user()` for the `/status/`, `/headers/` and `/user/` endpoints, in every output format (#63) @@ -57,7 +59,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `stocks.candles()` and `options.quotes()` retry each request on its own instead of the whole call: a failed chunk or symbol is re-issued alone, the healthy responses are kept, and one unreachable symbol no longer re-sends (and re-bills) every other symbol on each attempt (#83) - `options.expirations()` returns a `no_data` DataFrame with the same index as a populated one (`expirations` index, `updated` column), so concatenating results across symbols keeps the index name and gains no stray `expirations` column (#84) - The CSV output of `stocks.candles()` and `options.quotes()` is merged from the answers themselves: the file keeps the header the API sent (the requested `columns`, the human-readable names) and every row of every chunk or symbol; it used to compare each body against the model's field list and silently drop every body that differed, so a `columns=` request or a human-readable `options.quotes()` produced a header-only file, and an undecodable body (an HTML error page) vanished without a word. Such a body now raises `ParseError`, and so does a body the `csv` module cannot read (a field past its size limit), which escaped as a raw `csv.Error` (#86) -- A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model leaves it out, and its name is logged once per answer at DEBUG on `marketdata.logger`, where the call raised a bare `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` leave such a column out of their INTERNAL, DataFrame and JSON output, as they did, without the DEBUG line. Their CSV output keeps it, as the CSV of every other resource does; a header with none of the resource's columns, like an HTML error page, is still refused (#111) - DataFrame dates under `DateFormat.TIMESTAMP` and `DateFormat.SPREADSHEET` are the US/Eastern datetimes the default format gives, on pandas and polars. polars left `timestamp` datetimes as strings, a `timestamp` date (`2026-09-21`) landed at 20:00 the day before on both, and a spreadsheet serial, which the API writes as US/Eastern wall-clock time, was read as UTC, four or five hours early. A serial now reads to the second, without the float noise pandas kept (#143) - The `no_data` DataFrame and the header-only CSV carry the requested columns under a `columns=` filter, in request order, as a populated answer does; API names select the human-readable twins under `use_human_readable=True`, and under `add_headers=False` the empty CSV is an empty file. Known limit: the API's own aliases (`open` for `o`, `price`, `date`) are not translated, because the accepted set depends on the endpoint, so filtering on one of those still gives an empty frame of a different shape than the populated answer (#87) - A `404` `no_data` whose body is not JSON (a CDN or a proxy answering the 404 without one) returns the empty result on the JSON output too, instead of raising `ParseError` there while the other three formats returned their empty value: the output format never decides whether a call raises (#87) diff --git a/src/marketdata/__init__.py b/src/marketdata/__init__.py index a8dcdf3..acf8140 100644 --- a/src/marketdata/__init__.py +++ b/src/marketdata/__init__.py @@ -20,6 +20,7 @@ from marketdata.input_types.base import DateFormat, Mode, OutputFormat from marketdata.input_types.filters import DeltaFilter, StrikeFilter from marketdata.meta import ResponseMeta, get_meta +from marketdata.output_types.columns import get_extra from marketdata.types import UserRateLimits __all__ = [ @@ -32,6 +33,7 @@ "ResponseMeta", "UserRateLimits", "get_meta", + "get_extra", "BaseMarketdataException", "MarketdataHttpError", "BadRequestError", diff --git a/src/marketdata/output_types/columns.py b/src/marketdata/output_types/columns.py index d039964..87463ae 100644 --- a/src/marketdata/output_types/columns.py +++ b/src/marketdata/output_types/columns.py @@ -7,6 +7,71 @@ # By name, not `get_logger()`: importing the package must not attach a handler. _logger = logging.getLogger("marketdata.logger") +_EXTRA_ATTRIBUTE = "_marketdata_extra" + + +def get_extra(model: Any) -> dict[str, Any]: + """Read the columns of an API answer that the model built from it does not + declare. + + Args: + model: A model a resource call returned on ``OutputFormat.INTERNAL``, + one row of a list result, or a model built with ``from_dict``. + + Returns: + Each such column, named as the API sent it, with its value as the + model's answer was decoded: a number with a fraction is a ``Decimal`` + on the resources that decode money exactly. A single-object model + holds the whole column, a row its own value. Empty when there is none. + """ + namespace = getattr(model, "__dict__", None) + if not isinstance(namespace, dict): + return {} + return dict(namespace.get(_EXTRA_ATTRIBUTE, {})) + + +def _with_extra(model: Any, extra: dict[str, Any]) -> Any: + """Keep the undeclared columns of an answer on the model built from it. + + Args: + model: The model built from the answer. + extra: The columns the model does not declare, keyed as the API sent + them. + + Returns: + ``model``, which ``get_extra`` reads ``extra`` from. + """ + if extra: + model.__dict__[_EXTRA_ATTRIBUTE] = extra + return model + + +def _with_row_extra(models: list[Any], extra: dict[str, Any]) -> list[Any]: + """Keep the undeclared columns of a column-oriented answer on its rows. + + Args: + models: The models built from the answer's rows, in row order. + extra: The columns the model does not declare, keyed as the API sent + them. + + Returns: + ``models``. A column that is a list as long as the rows gives each row + its own value; any other value goes to every row as the API sent it. + """ + for index, model in enumerate(models): + _with_extra( + model, + { + key: ( + value[index] + if isinstance(value, list) and len(value) == len(models) + else value + ) + for key, value in extra.items() + }, + ) + return models + def _column_names(output_model: type) -> dict[str, str]: """Map each field of an output model to the name of its column in the API's @@ -56,8 +121,9 @@ def _column_types(output_model: type) -> dict[str, Any]: } -def _to_fields(output_model: type, data: Any) -> Any: - """Key an API answer by the fields of the output model it is built into. +def _split_fields(output_model: type, data: Any) -> tuple[Any, dict[str, Any]]: + """Split an API answer into the fields of the output model it is built into + and the columns the model does not declare. Args: output_model: The output model dataclass the answer is built into. @@ -65,23 +131,40 @@ def _to_fields(output_model: type, data: Any) -> Any: Returns: The values of the keys that name a column or a field of the model, - keyed by field name. Any other key is left out and logged at DEBUG, - except the status flag ``s``. An answer that is not an object, or that - carries none of the model's columns, comes back as it is, for the + keyed by field name, and every other key but the status flag ``s`` + with its value, as the API sent them. Those are logged at DEBUG, once + per answer. An answer that is not an object, or that carries none of + the model's columns, comes back as it is with no other key, for the model to refuse. """ columns = _column_names(output_model) if not isinstance(data, dict) or not any(key in data for key in columns.values()): - return data + return data, {} by_key = {field.name: field.name for field in fields(output_model)} by_key.update({column: name for name, column in columns.items()}) - undeclared = [key for key in data if key not in by_key and key != "s"] - if undeclared: + extra = { + key: value for key, value in data.items() if key not in by_key and key != "s" + } + if extra: _logger.debug( - f"The API sent the columns {undeclared!r}, which" - f" {output_model.__name__} does not declare; they are left out" + f"The API sent the columns {list(extra)!r}, which" + f" {output_model.__name__} does not declare; get_extra() holds them" ) - return {by_key[key]: value for key, value in data.items() if key in by_key} + return {by_key[key]: value for key, value in data.items() if key in by_key}, extra + + +def _to_fields(output_model: type, data: Any) -> Any: + """Key an API answer that carries only declared columns by the fields of + the output model it is built into. + + Args: + output_model: The output model dataclass the answer is built into. + data: The answer, keyed by column name. + + Returns: + The fields ``_split_fields`` returns. + """ + return _split_fields(output_model, data)[0] def _model_columns(output_model: type, requested: list[str] | None = None) -> list[str]: diff --git a/src/marketdata/output_types/stocks_earnings.py b/src/marketdata/output_types/stocks_earnings.py index 366d4a0..91a8ea5 100644 --- a/src/marketdata/output_types/stocks_earnings.py +++ b/src/marketdata/output_types/stocks_earnings.py @@ -3,7 +3,7 @@ from decimal import Decimal from typing import ClassVar -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_extra from marketdata.output_types.money import coerce_numbers from marketdata.utils import format_timestamp @@ -75,14 +75,15 @@ def from_dict(cls, data: dict) -> "StockEarnings": data: The answer, keyed as the API names its columns. Returns: - The earnings. A column the model does not declare is left out. + The earnings. A column the model does not declare is kept for ``get_extra``. Raises: TypeError: If a field is missing or a number field holds something that is not a number. ValueError: If a value cannot be read as its field's type. """ - return cls(**_to_fields(cls, data)) + fields, extra = _split_fields(cls, data) + return _with_extra(cls(**fields), extra) @dataclass @@ -143,5 +144,20 @@ def __str__(self) -> str: @classmethod def from_dict(cls, data: dict) -> "StockEarningsHumanReadable": - """Build the model from an API answer keyed by column name.""" - return cls(**_to_fields(cls, data)) + """Build the model from an API answer keyed by column name. + + Args: + data: The earnings, keyed as the API names its + human-readable columns. + + Returns: + The earnings. A column the model does not declare is + kept for ``get_extra``. + + Raises: + TypeError: If a field is missing or a number field holds something + that is not a number. + ValueError: If a value cannot be read as its field's type. + """ + fields, extra = _split_fields(cls, data) + return _with_extra(cls(**fields), extra) diff --git a/src/marketdata/output_types/stocks_prices.py b/src/marketdata/output_types/stocks_prices.py index c9fef80..65fb0e1 100644 --- a/src/marketdata/output_types/stocks_prices.py +++ b/src/marketdata/output_types/stocks_prices.py @@ -3,7 +3,7 @@ from decimal import Decimal from typing import ClassVar -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_extra from marketdata.output_types.money import coerce_numbers from marketdata.utils import format_timestamp @@ -42,14 +42,15 @@ def from_dict(cls, data: dict) -> "StockPrice": data: One price, keyed as the API names its columns. Returns: - The price. A column the model does not declare is left out. + The price. A column the model does not declare is kept for ``get_extra``. Raises: TypeError: If a field is missing or a number field holds something that is not a number. ValueError: If a value cannot be read as its field's type. """ - return cls(**_to_fields(cls, data)) + fields, extra = _split_fields(cls, data) + return _with_extra(cls(**fields), extra) @dataclass @@ -85,5 +86,20 @@ def __str__(self) -> str: @classmethod def from_dict(cls, data: dict) -> "StockPricesHumanReadable": - """Build the model from an API answer keyed by column name.""" - return cls(**_to_fields(cls, data)) + """Build the model from an API answer keyed by column name. + + Args: + data: One price, keyed as the API names its + human-readable columns. + + Returns: + The price. A column the model does not declare is + kept for ``get_extra``. + + Raises: + TypeError: If a field is missing or a number field holds something + that is not a number. + ValueError: If a value cannot be read as its field's type. + """ + fields, extra = _split_fields(cls, data) + return _with_extra(cls(**fields), extra) diff --git a/src/marketdata/output_types/stocks_quotes.py b/src/marketdata/output_types/stocks_quotes.py index 74ea015..a14ae3d 100644 --- a/src/marketdata/output_types/stocks_quotes.py +++ b/src/marketdata/output_types/stocks_quotes.py @@ -3,7 +3,7 @@ from decimal import Decimal from typing import ClassVar -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_extra from marketdata.output_types.money import coerce_numbers from marketdata.utils import format_timestamp @@ -53,14 +53,15 @@ def from_dict(cls, data: dict) -> "StockQuote": data: One quote, keyed as the API names its columns. Returns: - The quote. A column the model does not declare is left out. + The quote. A column the model does not declare is kept for ``get_extra``. Raises: TypeError: If a field is missing or a number field holds something that is not a number. ValueError: If a value cannot be read as its field's type. """ - return cls(**_to_fields(cls, data)) + fields, extra = _split_fields(cls, data) + return _with_extra(cls(**fields), extra) @dataclass @@ -108,5 +109,20 @@ def __str__(self) -> str: @classmethod def from_dict(cls, data: dict) -> "StockQuotesHumanReadable": - """Build the model from an API answer keyed by column name.""" - return cls(**_to_fields(cls, data)) + """Build the model from an API answer keyed by column name. + + Args: + data: One quote, keyed as the API names its + human-readable columns. + + Returns: + The quote. A column the model does not declare is + kept for ``get_extra``. + + Raises: + TypeError: If a field is missing or a number field holds something + that is not a number. + ValueError: If a value cannot be read as its field's type. + """ + fields, extra = _split_fields(cls, data) + return _with_extra(cls(**fields), extra) diff --git a/src/marketdata/resources/base.py b/src/marketdata/resources/base.py index c061646..909d9d3 100644 --- a/src/marketdata/resources/base.py +++ b/src/marketdata/resources/base.py @@ -1,3 +1,4 @@ +import csv from collections.abc import Callable, Iterator from contextlib import contextmanager from typing import TYPE_CHECKING, Any @@ -16,6 +17,9 @@ from marketdata.output_types.columns import _model_columns from marketdata.settings import settings from marketdata.utils import ( + _csv_records, + _unreadable_csv, + column_key, csv_header, parse_error, parse_json, @@ -73,6 +77,31 @@ def _parse_json_object(response: Response, *, exact: bool = False) -> dict[str, return data +def _check_csv_header( + response: Response, output_model: type, *, with_header: bool +) -> None: + """Refuse a CSV answer whose header names none of the resource's columns. + + Args: + response: The CSV answer. + output_model: The output model of the resource. + with_header: Whether the answer starts with a header row. + + Raises: + ParseError: If the answer starts with a header row that names none of + the model's columns, or with one the ``csv`` module cannot read. + """ + if not with_header: + return + try: + incoming = next(_csv_records(response.text), None) + except csv.Error as exc: + raise _unreadable_csv(response, exc) from exc + known = {column_key(name) for name in model_columns(output_model)} + if incoming is not None and not any(column_key(name) in known for name in incoming): + raise parse_error(response, f"unknown columns {incoming!r}") + + @contextmanager def model_errors(response: Response) -> Iterator[None]: """Turn a model refusing a value into a ``ParseError``. diff --git a/src/marketdata/resources/funds/candles.py b/src/marketdata/resources/funds/candles.py index 3fe79f7..5353a58 100644 --- a/src/marketdata/resources/funds/candles.py +++ b/src/marketdata/resources/funds/candles.py @@ -5,7 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.funds import FundsCandlesInput from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.funds_candles import FundsCandle, FundsCandlesHumanReadable from marketdata.params import universal_params from marketdata.resources.base import BaseResource, model_errors, no_data_result @@ -70,10 +70,11 @@ def candles( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = parse_json(response, exact=True) - data = get_data_records(_to_fields(output_model, data), exclude_keys=["s"]) + fields, extra = _split_fields(output_model, data) + data = get_data_records(fields, exclude_keys=["s"]) with model_errors(response): - return [output_model(**row) for row in data] + return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/markets/status.py b/src/marketdata/resources/markets/status.py index f26379f..49c97fc 100644 --- a/src/marketdata/resources/markets/status.py +++ b/src/marketdata/resources/markets/status.py @@ -5,7 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.markets import MarketStatusInput from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.markets_status import ( MarketStatus, MarketStatusHumanReadable, @@ -67,9 +67,10 @@ def status( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = parse_json(response) - data = get_data_records(_to_fields(output_model, data), exclude_keys=["s"]) + fields, extra = _split_fields(output_model, data) + data = get_data_records(fields, exclude_keys=["s"]) with model_errors(response): - return [output_model(**row) for row in data] + return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/options/chain.py b/src/marketdata/resources/options/chain.py index 4b6249b..ef6cdb1 100644 --- a/src/marketdata/resources/options/chain.py +++ b/src/marketdata/resources/options/chain.py @@ -5,7 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.options import OptionsChainInput from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_extra from marketdata.output_types.options_chain import ( OptionsChain, OptionsChainHumanReadable, @@ -13,6 +13,7 @@ from marketdata.params import universal_params from marketdata.resources.base import ( BaseResource, + _check_csv_header, _parse_json_object, model_errors, no_data_result, @@ -74,13 +75,19 @@ def chain( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = _parse_json_object(response, exact=True) + fields, extra = _split_fields(output_model, data) with model_errors(response): - return output_model(**_to_fields(output_model, data)) + return _with_extra(output_model(**fields), extra) elif user_universal_params.output_format == OutputFormat.JSON: return _parse_json_object(response) elif user_universal_params.output_format == OutputFormat.CSV: + _check_csv_header( + response, + output_model, + with_header=user_universal_params.add_headers is not False, + ) return user_universal_params.write_file(response.text) # This line should never be reached due to the universal_params decorator validating the output format diff --git a/src/marketdata/resources/options/expirations.py b/src/marketdata/resources/options/expirations.py index 322daec..0b24722 100644 --- a/src/marketdata/resources/options/expirations.py +++ b/src/marketdata/resources/options/expirations.py @@ -5,7 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.options import OptionsExpirationsInput from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_extra from marketdata.output_types.options_expirations import ( OptionsExpirations, OptionsExpirationsHumanReadable, @@ -13,6 +13,7 @@ from marketdata.params import universal_params from marketdata.resources.base import ( BaseResource, + _check_csv_header, _parse_json_object, model_errors, no_data_result, @@ -86,13 +87,19 @@ def expirations( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = _parse_json_object(response) + fields, extra = _split_fields(output_model, data) with model_errors(response): - return output_model(**_to_fields(output_model, data)) + return _with_extra(output_model(**fields), extra) elif user_universal_params.output_format == OutputFormat.JSON: return _parse_json_object(response) elif user_universal_params.output_format == OutputFormat.CSV: + _check_csv_header( + response, + output_model, + with_header=user_universal_params.add_headers is not False, + ) return user_universal_params.write_file(response.text) # This line should never be reached due to the universal_params decorator validating the output format diff --git a/src/marketdata/resources/options/lookup.py b/src/marketdata/resources/options/lookup.py index 9ccde9f..eb805f9 100644 --- a/src/marketdata/resources/options/lookup.py +++ b/src/marketdata/resources/options/lookup.py @@ -5,7 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.options import OptionsLookupInput from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_extra from marketdata.output_types.options_lookup import ( OptionsLookup, OptionsLookupHumanReadable, @@ -13,6 +13,7 @@ from marketdata.params import universal_params from marketdata.resources.base import ( BaseResource, + _check_csv_header, _parse_json_object, model_errors, no_data_result, @@ -78,13 +79,19 @@ def lookup( elif user_universal_params.output_format == OutputFormat.INTERNAL: data = _parse_json_object(response) + fields, extra = _split_fields(output_model, data) with model_errors(response): - return output_model(**_to_fields(output_model, data)) + return _with_extra(output_model(**fields), extra) elif user_universal_params.output_format == OutputFormat.JSON: return _parse_json_object(response) elif user_universal_params.output_format == OutputFormat.CSV: + _check_csv_header( + response, + output_model, + with_header=user_universal_params.add_headers is not False, + ) return user_universal_params.write_file(response.text) # This line should never be reached due to the universal_params decorator validating the output format diff --git a/src/marketdata/resources/stocks/earnings.py b/src/marketdata/resources/stocks/earnings.py index 15bde64..a51bc8f 100644 --- a/src/marketdata/resources/stocks/earnings.py +++ b/src/marketdata/resources/stocks/earnings.py @@ -12,6 +12,7 @@ from marketdata.params import universal_params from marketdata.resources.base import ( BaseResource, + _check_csv_header, _parse_json_object, model_errors, no_data_result, @@ -80,6 +81,11 @@ def earnings( return _parse_json_object(response) elif user_universal_params.output_format == OutputFormat.CSV: + _check_csv_header( + response, + output_model, + with_header=user_universal_params.add_headers is not False, + ) return user_universal_params.write_file(response.text) # This line should never be reached due to the universal_params decorator validating the output format diff --git a/src/marketdata/resources/stocks/news.py b/src/marketdata/resources/stocks/news.py index 4a5ef2e..4af3b2a 100644 --- a/src/marketdata/resources/stocks/news.py +++ b/src/marketdata/resources/stocks/news.py @@ -5,7 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.stocks import StocksNewsInput from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.stocks_news import StockNews, StockNewsHumanReadable from marketdata.params import universal_params from marketdata.resources.base import BaseResource, model_errors, no_data_result @@ -69,11 +69,10 @@ def news( ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = get_data_records( - _to_fields(output_model, parse_json(response)), exclude_keys=["s"] - ) + fields, extra = _split_fields(output_model, parse_json(response)) + data = get_data_records(fields, exclude_keys=["s"]) with model_errors(response): - return [output_model(**row) for row in data] + return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/stocks/prices.py b/src/marketdata/resources/stocks/prices.py index 615410b..ad58535 100644 --- a/src/marketdata/resources/stocks/prices.py +++ b/src/marketdata/resources/stocks/prices.py @@ -8,7 +8,7 @@ ) from marketdata.input_types.stocks import StocksPricesInput from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.stocks_prices import StockPrice, StockPricesHumanReadable from marketdata.params import universal_params from marketdata.resources.base import model_errors, no_data_result @@ -68,11 +68,10 @@ def prices( ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = get_data_records( - _to_fields(output_model, parse_json(response, exact=True)) - ) + fields, extra = _split_fields(output_model, parse_json(response, exact=True)) + data = get_data_records(fields) with model_errors(response): - return [output_model(**row) for row in data] + return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/stocks/quotes.py b/src/marketdata/resources/stocks/quotes.py index 026560d..639272d 100644 --- a/src/marketdata/resources/stocks/quotes.py +++ b/src/marketdata/resources/stocks/quotes.py @@ -5,7 +5,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.input_types.stocks import StocksQuotesInput from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.stocks_quotes import StockQuote, StockQuotesHumanReadable from marketdata.params import universal_params from marketdata.resources.base import BaseResource, model_errors, no_data_result @@ -65,12 +65,10 @@ def quotes( ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = get_data_records( - _to_fields(output_model, parse_json(response, exact=True)), - exclude_keys=["s"], - ) + fields, extra = _split_fields(output_model, parse_json(response, exact=True)) + data = get_data_records(fields, exclude_keys=["s"]) with model_errors(response): - return [output_model(**row) for row in data] + return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: return parse_json(response) diff --git a/src/marketdata/resources/utilities/base.py b/src/marketdata/resources/utilities/base.py index f054879..8536194 100644 --- a/src/marketdata/resources/utilities/base.py +++ b/src/marketdata/resources/utilities/base.py @@ -14,7 +14,7 @@ from marketdata.input_types.base import OutputFormat, UserUniversalAPIParams from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.resources.base import BaseResource, model_errors, no_data_result from marketdata.utils import dict_to_csv, get_data_records, is_no_data, parse_json @@ -70,9 +70,10 @@ def render( elif output_format == OutputFormat.INTERNAL: if as_records: - rows = get_data_records(_to_fields(output_model, data), exclude_keys=["s"]) + fields, extra = _split_fields(output_model, data) + rows = get_data_records(fields, exclude_keys=["s"]) with model_errors(response): - return [output_model(**row) for row in rows] + return _with_row_extra([output_model(**row) for row in rows], extra) with model_errors(response): return output_model.from_dict(data) diff --git a/src/tests/test_money_decimal.py b/src/tests/test_money_decimal.py index 15ba0f2..65df658 100644 --- a/src/tests/test_money_decimal.py +++ b/src/tests/test_money_decimal.py @@ -711,13 +711,37 @@ def _undeclared_columns_logged(caplog) -> list[str]: ] +SINGLE_REQUEST_CASES = [name for name in MODEL_CASES if name not in FAN_OUTS] + + +def _add_new_column(data: dict) -> list[str] | str: + """Add a column no model declares to an API answer. + + Args: + data: The answer, keyed by column name; changed in place. + + Returns: + The column's value: one text per row when the answer's first column + is a list, one text otherwise. + """ + first = data[next(key for key in data if key != "s")] + data["brandNew"] = ( + [f"new {index}" for index in range(len(first))] + if isinstance(first, list) + else "new" + ) + return data["brandNew"] + + @pytest.mark.parametrize("name", MODEL_CASES) -def test_a_column_no_model_declares_is_left_out_of_the_internal_result( +def test_a_column_no_model_declares_is_kept_for_get_extra( respx_mock, client, caplog, name ): """A column the API adds before the model declares it does not break the - call: the result is the one the answer builds without it, and the column - is named once at DEBUG, however many rows the answer has.""" + call: the fields are the ones the answer builds without it, ``get_extra`` + holds the column, a row its own value, and the column is named once at + DEBUG however many rows the answer has. The fan-outs merge only the + columns their model declares, so there it is gone and nothing is logged.""" case = MODEL_CASES[name] data = _load_fixture(case.fixture) _respond(respx_mock, case, json.dumps(data).encode()) @@ -725,19 +749,86 @@ def test_a_column_no_model_declares_is_left_out_of_the_internal_result( expected = case.call(client, output_format=OutputFormat.INTERNAL) assert _undeclared_columns_logged(caplog) == [] - data["brandNew"] = data[next(key for key in data if key != "s")] + column = _add_new_column(data) _respond(respx_mock, case, json.dumps(data).encode()) caplog.clear() with caplog.at_level("DEBUG", logger="marketdata.logger"): result = case.call(client, output_format=OutputFormat.INTERNAL) assert result == expected - model = type(result[0] if isinstance(result, list) else result).__name__ - logged = [ - f"The API sent the columns ['brandNew'], which {model} does not declare;" - " they are left out" + models = result if isinstance(result, list) else [result] + extras = [marketdata.get_extra(model) for model in models] + if name in FAN_OUTS: + assert extras == [{}] * len(models) + assert _undeclared_columns_logged(caplog) == [] + return + if isinstance(result, list) and isinstance(column, list): + assert extras == [{"brandNew": value} for value in column] + else: + assert extras == [{"brandNew": column}] * len(models) + assert _undeclared_columns_logged(caplog) == [ + f"The API sent the columns ['brandNew'], which {type(models[0]).__name__}" + " does not declare; get_extra() holds them" ] - assert _undeclared_columns_logged(caplog) == ([] if name in FAN_OUTS else logged) + + +@pytest.mark.parametrize("name", SINGLE_REQUEST_CASES) +def test_a_column_no_model_declares_reaches_every_format( + respx_mock, client, tmp_path, name +): + """An answer with a column the model does not declare carries it on every + format: INTERNAL through ``get_extra``, JSON and DataFrame as a column, + and CSV in the file, which is the API's text or, on the utilities, the + file written from the decoded answer.""" + case = MODEL_CASES[name] + data = _load_fixture(case.fixture) + _add_new_column(data) + _respond(respx_mock, case, json.dumps(data).encode()) + + result = case.call(client, output_format=OutputFormat.INTERNAL) + as_json = case.call(client, output_format=OutputFormat.JSON) + frame = case.call(client, output_format=OutputFormat.DATAFRAME) + if not name.startswith("utilities."): + first = next(key for key in data if key != "s") + _respond(respx_mock, case, f"{first},brandNew\r\nx,new 0\r\n".encode()) + path = case.call( + client, output_format=OutputFormat.CSV, filename=tmp_path / "answer.csv" + ) + + models = result if isinstance(result, list) else [result] + assert all("brandNew" in marketdata.get_extra(model) for model in models) + assert as_json["brandNew"] == data["brandNew"] + assert "brandNew" in frame.columns + header = pathlib.Path(path).read_text(encoding="utf-8").splitlines()[0] + assert "brandNew" in header.split(",") + + +@pytest.mark.parametrize( + "value", + ["new", ["one", "two", "three"]], + ids=["one value", "a list of another length"], +) +def test_an_undeclared_column_without_one_value_per_row_goes_to_every_row( + respx_mock, client, value +): + """A column the model does not declare that does not hold one value per + row is given to every row as the API sent it.""" + case = MODEL_CASES["funds.candles"] + data = _load_fixture(case.fixture) + data["brandNew"] = value + _respond(respx_mock, case, json.dumps(data).encode()) + + rows = case.call(client, output_format=OutputFormat.INTERNAL) + + assert len(rows) == 7 + assert [marketdata.get_extra(row) for row in rows] == [{"brandNew": value}] * 7 + + +def test_get_extra_is_empty_for_an_object_with_no_dict(): + """``get_extra`` finds nothing on an object that has no ``__dict__`` to + hold an undeclared column.""" + assert marketdata.get_extra(None) == {} + assert marketdata.get_extra(object()) == {} @pytest.mark.parametrize("name", MODEL_CASES) @@ -771,13 +862,13 @@ def test_an_object_with_none_of_the_model_columns_is_a_parse_error( "stocks.earnings human", ], ) -def test_a_body_that_is_not_an_object_is_one_parse_error_on_every_decoded_format( +def test_a_body_that_is_not_an_object_is_a_parse_error_on_every_format( respx_mock, client, tmp_path, name, body ): - """A single-object resource refuses a body that is not a JSON object with - the same ``ParseError`` on every format that decodes it, under the API's - names and the human-readable ones. The CSV file is the API's text as it - came, since that format decodes nothing.""" + """A single-object resource refuses a body that is not a JSON object on + every format, under the API's names and the human-readable ones: the same + ``ParseError`` on the three that decode it, and on CSV the one a header + with none of the resource's columns gets.""" case = MODEL_CASES[name] _respond(respx_mock, case, body) @@ -790,17 +881,70 @@ def test_a_body_that_is_not_an_object_is_one_parse_error_on_every_decoded_format with pytest.raises(ParseError) as failure: case.call(client, output_format=output_format) messages.add(failure.value.message) - path = case.call( - client, output_format=OutputFormat.CSV, filename=tmp_path / "answer.csv" - ) + with pytest.raises(ParseError) as refused: + case.call( + client, output_format=OutputFormat.CSV, filename=tmp_path / "answer.csv" + ) + text = body.decode() assert messages == { "Response body is not a valid answer of this resource (not a JSON object):" - f" {body.decode()!r}" + f" {text!r}" } + assert refused.value.message == ( + "Response body is not a valid answer of this resource (unknown columns" + f" [{text!r}]): {text!r}" + ) + assert not (tmp_path / "answer.csv").exists() + + +SINGLE_OBJECT_CASES = [ + "options.chain", + "options.expirations", + "options.lookup", + "stocks.earnings", +] + + +@pytest.mark.parametrize("name", SINGLE_OBJECT_CASES) +def test_a_csv_without_a_header_row_is_written_as_it_came( + respx_mock, client, tmp_path, name +): + """Under ``add_headers=False`` the answer has no header row to check, so + the file is the API's text as it came.""" + case = MODEL_CASES[name] + body = b"AAPL,200\r\n" + _respond(respx_mock, case, body) + + path = case.call( + client, + output_format=OutputFormat.CSV, + add_headers=False, + filename=tmp_path / "answer.csv", + ) + assert pathlib.Path(path).read_bytes() == body +def test_a_csv_header_the_csv_module_cannot_read_is_a_parse_error( + respx_mock, client, tmp_path +): + """A header past the ``csv`` module's field size limit cannot be read, so + the answer is refused and no file is written.""" + case = MODEL_CASES["options.chain"] + _respond(respx_mock, case, b"x" * (csv.field_size_limit() + 1)) + + with pytest.raises(ParseError) as refused: + case.call( + client, output_format=OutputFormat.CSV, filename=tmp_path / "answer.csv" + ) + + assert refused.value.message.startswith( + "Response body is not a valid answer of this resource (unreadable CSV: " + ) + assert not (tmp_path / "answer.csv").exists() + + @pytest.mark.parametrize("name", OTHER_DATE_KEYS.keys()) def test_a_date_the_model_cannot_read_is_a_parse_error_with_no_money_involved( respx_mock, client, name diff --git a/src/tests/test_options_chain.py b/src/tests/test_options_chain.py index 9b82572..bfe4b26 100644 --- a/src/tests/test_options_chain.py +++ b/src/tests/test_options_chain.py @@ -325,13 +325,16 @@ def test_get_options_chain_status_offline(load_json, respx_mock, client): def test_get_options_chain_response_200_csv(respx_mock, client): respx_mock.get("https://api.marketdata.app/v1/options/chain/AAPL/").respond( - text="AS RECEIVED FROM API", + text="optionSymbol,underlying,strike\nAAPL260116C00200000,AAPL,200\n", status_code=200, ) output = client.options.chain( "AAPL", output_format=OutputFormat.CSV, filename="test.csv" ) - assert pathlib.Path(output).read_text() == "AS RECEIVED FROM API" + assert ( + pathlib.Path(output).read_text() + == "optionSymbol,underlying,strike\nAAPL260116C00200000,AAPL,200\n" + ) def test_options_chain_input_date_range_aliases_on_wire(load_json, respx_mock, client): diff --git a/src/tests/test_options_expirations.py b/src/tests/test_options_expirations.py index 0ee0e8c..54db19f 100644 --- a/src/tests/test_options_expirations.py +++ b/src/tests/test_options_expirations.py @@ -333,10 +333,13 @@ def test_get_options_expirations_internal_ignores_a_column_filter(respx_mock, cl def test_get_options_expirations_response_200_csv(respx_mock, client): respx_mock.get("https://api.marketdata.app/v1/options/expirations/AAPL/").respond( - text="AS RECEIVED FROM API", + text="expirations,updated\n2026-01-16,1769000000\n", status_code=200, ) output = client.options.expirations( symbol="AAPL", output_format=OutputFormat.CSV, filename="test.csv" ) - assert pathlib.Path(output).read_text() == "AS RECEIVED FROM API" + assert ( + pathlib.Path(output).read_text() + == "expirations,updated\n2026-01-16,1769000000\n" + ) diff --git a/src/tests/test_options_lookup.py b/src/tests/test_options_lookup.py index c1e2fd5..42c062b 100644 --- a/src/tests/test_options_lookup.py +++ b/src/tests/test_options_lookup.py @@ -170,7 +170,7 @@ def test_get_options_lookup_response_200_csv(respx_mock, client): respx_mock.get( "https://api.marketdata.app/v1/options/lookup/AAPL 28-00-2023 200.0 call/" ).respond( - text="AS RECEIVED FROM API", + text="optionSymbol\nAAPL230728C00200000\n", status_code=200, ) output = client.options.lookup( @@ -178,4 +178,4 @@ def test_get_options_lookup_response_200_csv(respx_mock, client): output_format=OutputFormat.CSV, filename="test.csv", ) - assert pathlib.Path(output).read_text() == "AS RECEIVED FROM API" + assert pathlib.Path(output).read_text() == "optionSymbol\nAAPL230728C00200000\n" diff --git a/src/tests/test_stocks_earnings.py b/src/tests/test_stocks_earnings.py index 7540d16..63deb22 100644 --- a/src/tests/test_stocks_earnings.py +++ b/src/tests/test_stocks_earnings.py @@ -257,13 +257,16 @@ def test_get_stocks_earnings_status_offline(respx_mock, client): def test_get_stocks_earnings_response_200_csv(respx_mock, client): respx_mock.get("https://api.marketdata.app/v1/stocks/earnings/AAPL/").respond( - text="AS RECEIVED FROM API", + text="symbol,fiscalYear,fiscalQuarter\nAAPL,2026,1\n", status_code=200, ) output = client.stocks.earnings( symbol="AAPL", output_format=OutputFormat.CSV, filename="test.csv" ) - assert pathlib.Path(output).read_text() == "AS RECEIVED FROM API" + assert ( + pathlib.Path(output).read_text() + == "symbol,fiscalYear,fiscalQuarter\nAAPL,2026,1\n" + ) def test_earnings_internal_reads_timestamp_dates_as_us_eastern_midnight( diff --git a/src/tests/test_stocks_prices.py b/src/tests/test_stocks_prices.py index 15691ae..b92ba64 100644 --- a/src/tests/test_stocks_prices.py +++ b/src/tests/test_stocks_prices.py @@ -7,6 +7,7 @@ import pytest import pytz +import marketdata from marketdata.exceptions import ParseError, ServerError from marketdata.input_types.base import OutputFormat from marketdata.output_types.stocks_prices import StockPrice, StockPricesHumanReadable @@ -49,9 +50,9 @@ def test_stock_prices_human_readable_str(): assert isinstance(str(instance), str) -def test_stock_price_from_dict_leaves_out_an_undeclared_column(): +def test_stock_price_from_dict_keeps_an_undeclared_column_for_get_extra(): """StockPrice.from_dict builds the price from one row keyed by the API's - names and leaves out a column the model does not declare.""" + names and keeps a column the model does not declare for get_extra.""" instance = StockPrice.from_dict( { "s": "ok", @@ -72,11 +73,13 @@ def test_stock_price_from_dict_leaves_out_an_undeclared_column(): changepct=-0.0024, updated=1765478200, ) + assert marketdata.get_extra(instance) == {"brandNew": 1} -def test_stock_prices_human_readable_from_dict_leaves_out_an_undeclared_column(): +def test_stock_prices_human_readable_from_dict_keeps_an_undeclared_column(): """StockPricesHumanReadable.from_dict builds the price from one row keyed by - the human-readable names and leaves out a column the model does not declare.""" + the human-readable names and keeps a column the model does not declare for + get_extra.""" instance = StockPricesHumanReadable.from_dict( { "Symbol": "AAPL", @@ -95,6 +98,7 @@ def test_stock_prices_human_readable_from_dict_leaves_out_an_undeclared_column() Change_Percent=-0.0024, Date=1765478200, ) + assert marketdata.get_extra(instance) == {"brandNew": 1} def test_get_stocks_prices_response_200_internal(load_json, respx_mock, client): From bbeea3cb051c942977ddb808c439aae464739141 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 13:36:15 -0300 Subject: [PATCH 06/13] docs(utilities): document render and the CSV tests of the single-object resources --- src/marketdata/resources/utilities/base.py | 21 ++++++++++++++++++--- src/tests/test_options_chain.py | 2 ++ src/tests/test_options_expirations.py | 2 ++ src/tests/test_options_lookup.py | 2 ++ src/tests/test_stocks_earnings.py | 2 ++ 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/marketdata/resources/utilities/base.py b/src/marketdata/resources/utilities/base.py index 8536194..02f8702 100644 --- a/src/marketdata/resources/utilities/base.py +++ b/src/marketdata/resources/utilities/base.py @@ -45,10 +45,25 @@ def render( as_records: bool, index_columns: list[str] | None = None, ): - """Turn the response into the requested output format. + """Turn a utilities answer into the requested output format. - ``as_records`` distinguishes column-oriented payloads (one row per entry, - like ``/status/``) from flat objects (``/headers/``, ``/user/``). + Args: + user_universal_params: The call's universal parameters. + response: The API's answer. + output_model: The output model of the resource. + as_records: Whether the answer is column-oriented, one row per entry + (``/status/``), rather than a flat object (``/headers/``, + ``/user/``). + index_columns: The index columns of the DataFrame. + + Returns: + The rows or the model on INTERNAL, each row keeping the columns its + model does not declare for ``get_extra``; the decoded answer on JSON; + a DataFrame; or the path of the CSV file, written from the decoded + answer. The empty result for a ``no_data`` answer. + + Raises: + ParseError: If the body is not valid JSON or the model refuses it. """ output_format = user_universal_params.output_format diff --git a/src/tests/test_options_chain.py b/src/tests/test_options_chain.py index bfe4b26..3b2fe37 100644 --- a/src/tests/test_options_chain.py +++ b/src/tests/test_options_chain.py @@ -324,6 +324,8 @@ def test_get_options_chain_status_offline(load_json, respx_mock, client): def test_get_options_chain_response_200_csv(respx_mock, client): + """A CSV answer whose header names the resource's columns is written to + the file as it came.""" respx_mock.get("https://api.marketdata.app/v1/options/chain/AAPL/").respond( text="optionSymbol,underlying,strike\nAAPL260116C00200000,AAPL,200\n", status_code=200, diff --git a/src/tests/test_options_expirations.py b/src/tests/test_options_expirations.py index 54db19f..d9c37a3 100644 --- a/src/tests/test_options_expirations.py +++ b/src/tests/test_options_expirations.py @@ -332,6 +332,8 @@ def test_get_options_expirations_internal_ignores_a_column_filter(respx_mock, cl def test_get_options_expirations_response_200_csv(respx_mock, client): + """A CSV answer whose header names the resource's columns is written to + the file as it came.""" respx_mock.get("https://api.marketdata.app/v1/options/expirations/AAPL/").respond( text="expirations,updated\n2026-01-16,1769000000\n", status_code=200, diff --git a/src/tests/test_options_lookup.py b/src/tests/test_options_lookup.py index 42c062b..8332525 100644 --- a/src/tests/test_options_lookup.py +++ b/src/tests/test_options_lookup.py @@ -167,6 +167,8 @@ def test_get_options_lookup_status_offline(respx_mock, client): def test_get_options_lookup_response_200_csv(respx_mock, client): + """A CSV answer whose header names the resource's columns is written to + the file as it came.""" respx_mock.get( "https://api.marketdata.app/v1/options/lookup/AAPL 28-00-2023 200.0 call/" ).respond( diff --git a/src/tests/test_stocks_earnings.py b/src/tests/test_stocks_earnings.py index 63deb22..d0e147d 100644 --- a/src/tests/test_stocks_earnings.py +++ b/src/tests/test_stocks_earnings.py @@ -256,6 +256,8 @@ def test_get_stocks_earnings_status_offline(respx_mock, client): def test_get_stocks_earnings_response_200_csv(respx_mock, client): + """A CSV answer whose header names the resource's columns is written to + the file as it came.""" respx_mock.get("https://api.marketdata.app/v1/stocks/earnings/AAPL/").respond( text="symbol,fiscalYear,fiscalQuarter\nAAPL,2026,1\n", status_code=200, From 548338cf9aca2232f82b34e5cfccc570fb426b37 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 14:34:07 -0300 Subject: [PATCH 07/13] feat(resources)!: carry undeclared columns through the fan-out merge stocks.candles and options.quotes merge every column their answers send, not only the ones their model declares: a key other than the status flag that holds a list is a column, and every answer must carry it as a list as long as its rows, the rule a declared column follows. INTERNAL keeps such a column for get_extra, each row's value on candles and the whole column on option quotes, and JSON and DATAFRAME carry it by the API's name, as the merged CSV already did. A CSV answer without a header row, on the four single-object resources and in the fan-out merge, is refused when it reads as JSON or HTML, since no CSV row of these resources starts with `[`, `{` or `<`. BREAKING CHANGE: a fan-out answer that lacks a column another answer sends, declared or not, or sends it with another length, fails the call with ParseError on every format, where INTERNAL, JSON and DATAFRAME left an undeclared column out. Under add_headers=False, the CSV of options.chain, options.expirations, options.lookup, stocks.earnings, stocks.candles and options.quotes raises ParseError for a body that reads as JSON or HTML, where it wrote it to the file. --- CHANGELOG.md | 4 +- src/marketdata/output_types/columns.py | 14 --- src/marketdata/resources/base.py | 6 +- src/marketdata/resources/options/quotes.py | 7 +- src/marketdata/resources/stocks/candles.py | 13 ++- src/marketdata/utils.py | 69 ++++++++----- src/tests/test_money_decimal.py | 109 +++++++++++++++++---- src/tests/test_options_quotes.py | 36 +++++++ src/tests/test_utils.py | 16 +-- 9 files changed, 200 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a2b81..9279385 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,10 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Money in the `OutputFormat.INTERNAL` models is `decimal.Decimal` instead of `float`, built from the digits in the response body, which those resources now decode with `parse_float=Decimal` on the INTERNAL path: `ask`, `bid`, `mid`, `last`, `change` of `StockQuote`, `mid` and `change` of `StockPrice`, `o`, `h`, `l`, `c` of `StockCandle` and `FundsCandle`, the three EPS lists of `StockEarnings`, `strike`, `bid`, `mid`, `ask`, `last`, `intrinsicValue`, `extrinsicValue`, `underlyingPrice` of `OptionsChain` and `OptionsQuotes`, and the same fields of every human-readable twin. Greeks, IV and percentages stay `float`, sizes and counts `int`; DataFrames (pandas and polars), JSON and CSV output keep the plain float parse. Migration: arithmetic mixing `Decimal` and `float` raises `TypeError`, and a comparison with a float literal compares against its binary value (`Decimal("65.1") == 65.1` is `False`), so use `Decimal` literals or `int`; `float(value)` gives a float back, and `OutputFormat.DATAFRAME` is the float path for analysis (a pandas DataFrame built from models gets `object` columns, and `json.dumps(..., default=str)` writes the money as strings). A model built by hand converts its money arguments: a float or a numpy scalar through its shortest repr (`65.1` is `Decimal("65.1")`), a NaN number, which is how pandas writes a missing value, to `None`; a value that is not a number, a string that reads as NaN, or an infinity raises and names the field, while an array (anything that is not a string and has `__array__` and `__len__`: a numpy array, a pandas or polars Series, a pandas Index) is left as it came, and a `dict`, a `set` or a generator raises. On the API path the same refusal is a `ParseError` naming the field and the body, not a bare `TypeError` (#50) - Dates on `OutputFormat.INTERNAL` are read with the API's own rule and always come back as US/Eastern datetimes. A number from 10000 up to 200000 is a spreadsheet serial of US/Eastern wall-clock time, from 200000 Unix seconds, from `1e10` milliseconds and from `1e13` nanoseconds. A number under 10000 is a relative range for the API, not a date, and raises `ParseError`. The SDK used a boundary of its own at 60000, so from April 2064 a serial would have read as 1970. A `timestamp` date (`2026-09-21`) and a spreadsheet serial are now midnight and wall-clock time in US/Eastern, where they came back as naive datetimes, and a serial is rounded to the second. The intraday `from_date` and `to_date` of `stocks.candles()` are read only when they are ISO dates or numbers the API reads as dates (a spreadsheet serial or a Unix time), which the SDK splits into one request per year: a relative range or a keyword goes to the API as it is, in one request, where `"60"` was sent as 1900-02-28 and `"yesterday"` was refused. `UserRateLimits.reset_time` takes a datetime or Unix seconds, and refuses anything else (#116). Migration: compare INTERNAL dates against aware datetimes (`pytz.timezone("US/Eastern").localize(...)`), not naive ones - A response body carrying a `NaN`, `Infinity` or `-Infinity` literal raises `ParseError` on every output format that decodes it, where `INTERNAL`, `JSON` and `DATAFRAME` used to return it as `nan` or `inf`. Those literals are not JSON, and the renderer behind the API's data endpoints refuses them, answering `500`, so one in their bodies was written by something in between (`/status/` and `/headers/` are served elsewhere). `CSV` output writes the API's body as it came, since it reads no number, except for `client.utilities`, which builds its CSV from the decoded body and refuses it too. A number past what a float holds is not refused: `JSON` and `DATAFRAME` read `1e400` as `inf` while `INTERNAL` keeps it exact, and `INTERNAL` refuses one past what a `Decimal` holds. That renderer does not write such a number, and refusing it on the float path would slow every decode. Migration: code that read `nan` or `inf` from such a body catches `ParseError` (or `BaseMarketdataException`) instead; no setting brings the literals back (#50) -- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`), and their CSV output raises `ParseError` too, since it now refuses a header row with none of the resource's columns, where it wrote the body to the file. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) +- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`), and their CSV output raises `ParseError` too, since it now refuses a header row with none of the resource's columns and, under `add_headers=False`, a body that reads as JSON or HTML, where it wrote the body to the file. The CSV output of `stocks.candles()` and `options.quotes()` under `add_headers=False` refuses such a body the same way. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) - `options.chain(delta=...)` checks a number before it sends it, the way it checks `strike`, and keeps it as the text it sends. A `bool`, a NaN, an infinity, a number a float cannot hold and a delta outside -1 to 1 now raise `ValidationError` before any request, where they used to be sent: `delta=nan` came back as the whole chain, since the API reads it as no filter, `delta=True` was sent as `1.0`, and `delta=30` was sent as `30.0`. `OptionsChainInput.delta` holds that text (`"0.5"`) instead of a `float` (#101). Migration: pass a finite number from -1 to 1, the way the public docs write a delta (`0.3`), a `DeltaFilter`, or `delta=None` for no filter; code that reads `OptionsChainInput(...).delta` gets the text sent to the API, and `float()` turns a plain number back. - Every date field of the `OutputFormat.INTERNAL` models admits `None` in its annotation: `datetime.datetime | None` for one date, `list[datetime.datetime | None]` for a column of them, and `datetime.date | None` for `MarketStatus.date`, as `OptionsExpirations.updated` already did. A `null` date in an answer reads as `None` in its own row (see Fixed), and the annotations now say so. Migration: a type checker now asks for a `None` check before a `datetime` method is called on a date field (`if quote.updated is not None: quote.updated.date()`). At run time nothing that used to return changes: an answer with a `null` date used to raise `ParseError`, or on `OptionsQuotes` shift the column (#106) -- A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model is built without it, where the call raised a bare `TypeError`, and `marketdata.get_extra(model)` returns it as the API sent it: the whole column on a single-object model, the row's own value on each row of a list result. The column is named once per answer at DEBUG on `marketdata.logger`. `StockEarnings.from_dict()`, `StockPrice.from_dict()`, `StockQuote.from_dict()` and their human-readable twins keep such a key for `get_extra()` the same way, where they raised `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` still leave such a column out of their INTERNAL, DataFrame and JSON output when they merge, with nothing for `get_extra()`; their CSV output keeps it, where it raised `ParseError`, because `merge_csv_responses()` now refuses only a header with none of the resource's columns. Migration: code that caught the `TypeError` or `ParseError` an added column raised now gets the result; read the column with `get_extra()` until the model declares it (#111) +- A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model is built without it, where the call raised a bare `TypeError`, and `marketdata.get_extra(model)` returns it as the API sent it: the whole column on a single-object model, the row's own value on each row of a list result. The column is named once per answer at DEBUG on `marketdata.logger`. `StockEarnings.from_dict()`, `StockPrice.from_dict()`, `StockQuote.from_dict()` and their human-readable twins keep such a key for `get_extra()` the same way, where they raised `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` carry such a column through their merge on every format, INTERNAL through `get_extra()`, as long as every chunk or symbol sends it: like a declared column, an answer without it, or with a list of another length, fails the call with `ParseError` on every format. Their CSV output keeps it, where it raised `ParseError`, because `merge_csv_responses()` now refuses only a header with none of the resource's columns. Migration: code that caught the `TypeError` or `ParseError` an added column raised now gets the result; read the column with `get_extra()` until the model declares it (#111) ### Changed diff --git a/src/marketdata/output_types/columns.py b/src/marketdata/output_types/columns.py index 87463ae..15e12bf 100644 --- a/src/marketdata/output_types/columns.py +++ b/src/marketdata/output_types/columns.py @@ -153,20 +153,6 @@ def _split_fields(output_model: type, data: Any) -> tuple[Any, dict[str, Any]]: return {by_key[key]: value for key, value in data.items() if key in by_key}, extra -def _to_fields(output_model: type, data: Any) -> Any: - """Key an API answer that carries only declared columns by the fields of - the output model it is built into. - - Args: - output_model: The output model dataclass the answer is built into. - data: The answer, keyed by column name. - - Returns: - The fields ``_split_fields`` returns. - """ - return _split_fields(output_model, data)[0] - - def _model_columns(output_model: type, requested: list[str] | None = None) -> list[str]: """List the columns a result of an output model carries. diff --git a/src/marketdata/resources/base.py b/src/marketdata/resources/base.py index 909d9d3..a4699d5 100644 --- a/src/marketdata/resources/base.py +++ b/src/marketdata/resources/base.py @@ -18,6 +18,7 @@ from marketdata.settings import settings from marketdata.utils import ( _csv_records, + _reads_as_json_or_html, _unreadable_csv, column_key, csv_header, @@ -89,9 +90,12 @@ def _check_csv_header( Raises: ParseError: If the answer starts with a header row that names none of - the model's columns, or with one the ``csv`` module cannot read. + the model's columns, or with one the ``csv`` module cannot read, + or, without a header row, if it reads as JSON or HTML. """ if not with_header: + if _reads_as_json_or_html(response.text): + raise parse_error(response, "JSON or HTML, not CSV") return try: incoming = next(_csv_records(response.text), None) diff --git a/src/marketdata/resources/options/quotes.py b/src/marketdata/resources/options/quotes.py index 46a6218..a62f8a2 100644 --- a/src/marketdata/resources/options/quotes.py +++ b/src/marketdata/resources/options/quotes.py @@ -11,7 +11,7 @@ from marketdata.input_types.options import OptionsQuotesInput from marketdata.internal_settings import MAX_CONCURRENT_REQUESTS, VALID_STATUS_CODES from marketdata.output_handlers import get_dataframe_output_handler -from marketdata.output_types.columns import _to_fields +from marketdata.output_types.columns import _split_fields, _with_extra from marketdata.output_types.options_quotes import ( OptionsQuotes, OptionsQuotesHumanReadable, @@ -139,10 +139,11 @@ def _model_alone(response: Response) -> object: [response], alone, output_model.answer_keys() ) joined = output_model.join_dicts(alone, keys) - return output_model(**_to_fields(output_model, joined)) + return output_model(**_split_fields(output_model, joined)[0]) + fields, extra = _split_fields(output_model, data) with merged_model_errors(usable, _model_alone): - return output_model(**_to_fields(output_model, data)) + return _with_extra(output_model(**fields), extra) if user_universal_params.output_format == OutputFormat.JSON: return data diff --git a/src/marketdata/resources/stocks/candles.py b/src/marketdata/resources/stocks/candles.py index 5025e66..ed846b8 100644 --- a/src/marketdata/resources/stocks/candles.py +++ b/src/marketdata/resources/stocks/candles.py @@ -12,6 +12,7 @@ from marketdata.input_types.stocks import StocksCandlesInput from marketdata.internal_settings import MAX_CONCURRENT_REQUESTS from marketdata.output_handlers import get_dataframe_output_handler +from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.stocks_candles import ( StockCandle, StockCandlesHumanReadable, @@ -161,16 +162,20 @@ def _get_responses_data( ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = _get_responses_data(responses, exact=True) - data = get_data_records(data, exclude_keys=["s"]) + fields, extra = _split_fields( + output_model, _get_responses_data(responses, exact=True) + ) + data = get_data_records(fields, exclude_keys=["s"]) def _rows_alone(response: httpx.Response) -> list: - alone = _get_responses_data([response], exact=True) + alone = _split_fields( + output_model, _get_responses_data([response], exact=True) + )[0] rows = get_data_records(alone, exclude_keys=["s"]) return [output_model(**row) for row in rows] with merged_model_errors(responses, _rows_alone): - return [output_model(**row) for row in data] + return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: data = _get_responses_data(responses) diff --git a/src/marketdata/utils.py b/src/marketdata/utils.py index 85cbd6d..f371b79 100644 --- a/src/marketdata/utils.py +++ b/src/marketdata/utils.py @@ -368,7 +368,8 @@ def merge_csv_responses( the same column still merges. A column of the header that matches none of them is kept. with_header: Whether the bodies start with a header row. Without one, - only the row width is checked, against the first row seen. + only the row width is checked, against the first row seen, and a + body that reads as JSON or HTML is refused. Returns: The first answer's header, if any, then every row of every answer, @@ -379,8 +380,8 @@ def merge_csv_responses( ParseError: Naming the offending response, for a body the ``csv`` module cannot read, a body with no header row, a header with none of the resource's columns or with other columns, or in another - order, than the first answer's, or a row not as wide as the - header. + order, than the first answer's, a body without a header row that + reads as JSON or HTML, or a row not as wide as the header. """ known = {column_key(name) for name in known_columns} header: list[str] | None = None @@ -390,6 +391,8 @@ def merge_csv_responses( writer = csv.writer(output) for response in responses: + if not with_header and _reads_as_json_or_html(response.text): + raise parse_error(response, "JSON or HTML, not CSV") rows = _csv_records(response.text) try: if with_header: @@ -441,26 +444,44 @@ def _unreadable_csv(response: Response, error: csv.Error) -> ParseError: return parse_error(response, f"unreadable CSV: {error}") +def _reads_as_json_or_html(text: str) -> bool: + """Tell whether a CSV answer is a JSON value or an HTML page instead. + + Args: + text: The body. + + Returns: + Whether, past a BOM and white space, it starts with ``[``, ``{`` or + ``<``, or is ``null``. + """ + body = text.lstrip(BOM).strip() + return body[:1] in ("[", "{", "<") or body == "null" + + def json_answer_columns( responses: list[Response], answers: list[Any], keys: list[str] ) -> list[str]: - """The columns a fan-out merges from its decoded JSON answers (#90). - - They are the ``keys`` (the model's columns) that any answer carries, keys - the model does not know being left out. Since every answer must carry - them all, a successful merge has the first answer's columns in the first - answer's order: under ``columns=`` the API sends the requested columns - only, in the order they were requested, which is also the order of the - empty result and of every single-request resource. Every answer must be a - JSON object - carrying all of them as lists of one length, since the merge concatenates - column by column and a missing or short column would shift every later - row into the wrong symbol or chunk. Anything else raises ``ParseError`` - naming the offending response: a body that is not an object (``null``, a - list), answers with none of the keys (a proxy's JSON error page), an - answer missing a column another one carries, whichever it is, or a column - that is not a list of the same length as the others. A merge with no - columns would read as "no data" (#82). + """List the columns a fan-out merges from its decoded JSON answers. + + Every answer must carry every column as a list, all of one length, since + the merge concatenates column by column and a missing or short column + would shift every later row into the wrong symbol or chunk. + + Args: + responses: The answers, in merge order. + answers: Their decoded bodies. + keys: The model's columns. + + Returns: + The ``keys`` any answer carries, and every other key but the status + flag ``s`` that holds a list, in the order the answers first carry + them: under ``columns=``, the order the columns were requested in. + + Raises: + ParseError: Naming the offending response, for a body that is not a + JSON object, answers with none of ``keys`` (a proxy's JSON error + page), an answer missing a column another one carries, or a + column that is not a list as long as the others. """ for response, answer in zip(responses, answers, strict=True): if not isinstance(answer, dict): @@ -468,10 +489,12 @@ def json_answer_columns( known = set(keys) columns: list[str] = [] for answer in answers: - for key in answer: - if key in known and key not in columns: + for key, value in answer.items(): + if key not in columns and ( + key in known or (key != "s" and isinstance(value, list)) + ): columns.append(key) - if not columns: + if not any(key in known for key in columns): raise parse_error(responses[0], "none of this resource's fields") for response, answer in zip(responses, answers): missing = [key for key in columns if key not in answer] diff --git a/src/tests/test_money_decimal.py b/src/tests/test_money_decimal.py index 65df658..9cc0fb7 100644 --- a/src/tests/test_money_decimal.py +++ b/src/tests/test_money_decimal.py @@ -7,6 +7,7 @@ """ import array +import copy import csv import dataclasses import datetime @@ -15,6 +16,7 @@ import json import math import pathlib +import pickle import pkgutil import typing from decimal import Decimal, InvalidOperation, localcontext @@ -694,13 +696,6 @@ def test_a_body_missing_a_column_the_model_needs_is_a_parse_error( for name, case in {**MONEY_CASES, **OTHER_CASES}.items() if name not in ("utilities.headers", "utilities.user") } -# They merge only the columns their model declares, before any model is built. -FAN_OUTS = ( - "stocks.candles", - "stocks.candles human", - "options.quotes", - "options.quotes human", -) def _undeclared_columns_logged(caplog) -> list[str]: @@ -711,9 +706,6 @@ def _undeclared_columns_logged(caplog) -> list[str]: ] -SINGLE_REQUEST_CASES = [name for name in MODEL_CASES if name not in FAN_OUTS] - - def _add_new_column(data: dict) -> list[str] | str: """Add a column no model declares to an API answer. @@ -740,8 +732,8 @@ def test_a_column_no_model_declares_is_kept_for_get_extra( """A column the API adds before the model declares it does not break the call: the fields are the ones the answer builds without it, ``get_extra`` holds the column, a row its own value, and the column is named once at - DEBUG however many rows the answer has. The fan-outs merge only the - columns their model declares, so there it is gone and nothing is logged.""" + DEBUG however many rows the answer has, the fan-outs' merged answer + included.""" case = MODEL_CASES[name] data = _load_fixture(case.fixture) _respond(respx_mock, case, json.dumps(data).encode()) @@ -758,10 +750,6 @@ def test_a_column_no_model_declares_is_kept_for_get_extra( assert result == expected models = result if isinstance(result, list) else [result] extras = [marketdata.get_extra(model) for model in models] - if name in FAN_OUTS: - assert extras == [{}] * len(models) - assert _undeclared_columns_logged(caplog) == [] - return if isinstance(result, list) and isinstance(column, list): assert extras == [{"brandNew": value} for value in column] else: @@ -772,14 +760,14 @@ def test_a_column_no_model_declares_is_kept_for_get_extra( ] -@pytest.mark.parametrize("name", SINGLE_REQUEST_CASES) +@pytest.mark.parametrize("name", MODEL_CASES) def test_a_column_no_model_declares_reaches_every_format( respx_mock, client, tmp_path, name ): """An answer with a column the model does not declare carries it on every - format: INTERNAL through ``get_extra``, JSON and DataFrame as a column, - and CSV in the file, which is the API's text or, on the utilities, the - file written from the decoded answer.""" + format, the fan-outs' merge included: INTERNAL through ``get_extra``, JSON + and DataFrame as a column, and CSV in the file, which is the API's text + or, on the utilities, the file written from the decoded answer.""" case = MODEL_CASES[name] data = _load_fixture(case.fixture) _add_new_column(data) @@ -868,7 +856,8 @@ def test_a_body_that_is_not_an_object_is_a_parse_error_on_every_format( """A single-object resource refuses a body that is not a JSON object on every format, under the API's names and the human-readable ones: the same ``ParseError`` on the three that decode it, and on CSV the one a header - with none of the resource's columns gets.""" + with none of the resource's columns gets or, without a header row, the one + a body that reads as JSON gets.""" case = MODEL_CASES[name] _respond(respx_mock, case, body) @@ -885,6 +874,13 @@ def test_a_body_that_is_not_an_object_is_a_parse_error_on_every_format( case.call( client, output_format=OutputFormat.CSV, filename=tmp_path / "answer.csv" ) + with pytest.raises(ParseError) as headerless: + case.call( + client, + output_format=OutputFormat.CSV, + add_headers=False, + filename=tmp_path / "answer.csv", + ) text = body.decode() assert messages == { @@ -895,6 +891,10 @@ def test_a_body_that_is_not_an_object_is_a_parse_error_on_every_format( "Response body is not a valid answer of this resource (unknown columns" f" [{text!r}]): {text!r}" ) + assert headerless.value.message == ( + "Response body is not a valid answer of this resource (JSON or HTML, not" + f" CSV): {text!r}" + ) assert not (tmp_path / "answer.csv").exists() @@ -926,6 +926,73 @@ def test_a_csv_without_a_header_row_is_written_as_it_came( assert pathlib.Path(path).read_bytes() == body +@pytest.mark.parametrize( + "body", + [b'{"error": "upstream timeout"}', b"502 Bad Gateway"], + ids=["a JSON object", "an HTML page"], +) +def test_a_headerless_csv_that_reads_as_json_or_html_is_a_parse_error( + respx_mock, client, tmp_path, body +): + """Under ``add_headers=False`` a body that reads as JSON or HTML is not a + CSV answer, so it is refused and no file is written.""" + case = MODEL_CASES["options.chain"] + _respond(respx_mock, case, body) + + with pytest.raises(ParseError) as refused: + case.call( + client, + output_format=OutputFormat.CSV, + add_headers=False, + filename=tmp_path / "answer.csv", + ) + + assert "(JSON or HTML, not CSV)" in refused.value.message + assert not (tmp_path / "answer.csv").exists() + + +@pytest.mark.parametrize("name", ["stocks.candles", "options.quotes"]) +def test_a_fan_out_headerless_csv_that_reads_as_json_is_a_parse_error( + respx_mock, client, tmp_path, name +): + """The fan-outs' merge refuses a headerless body that reads as JSON, as + their JSON output refuses a body that is not an object.""" + case = MODEL_CASES[name] + _respond(respx_mock, case, b"[1]") + + with pytest.raises(ParseError) as refused: + case.call( + client, + output_format=OutputFormat.CSV, + add_headers=False, + filename=tmp_path / "answer.csv", + ) + with pytest.raises(ParseError) as decoded: + case.call(client, output_format=OutputFormat.JSON) + + assert "(JSON or HTML, not CSV)" in refused.value.message + assert "(not a JSON object)" in decoded.value.message + assert not (tmp_path / "answer.csv").exists() + + +def test_get_extra_survives_copy_and_pickle(respx_mock, client): + """The columns a model does not declare live on the model itself, so a + copy, a deep copy and a pickled model keep them.""" + case = MODEL_CASES["stocks.quotes"] + data = _load_fixture(case.fixture) + _add_new_column(data) + _respond(respx_mock, case, json.dumps(data).encode()) + + quote = case.call(client, output_format=OutputFormat.INTERNAL)[0] + + for twin in ( + copy.copy(quote), + copy.deepcopy(quote), + pickle.loads(pickle.dumps(quote)), + ): + assert marketdata.get_extra(twin) == {"brandNew": "new 0"} + + def test_a_csv_header_the_csv_module_cannot_read_is_a_parse_error( respx_mock, client, tmp_path ): diff --git a/src/tests/test_options_quotes.py b/src/tests/test_options_quotes.py index e39afb5..1a7aa2c 100644 --- a/src/tests/test_options_quotes.py +++ b/src/tests/test_options_quotes.py @@ -896,6 +896,42 @@ def test_options_quotes_csv_keeps_a_column_the_api_adds(respx_mock, client, tmp_ ) +def test_options_quotes_an_added_column_one_symbol_lacks_is_a_parse_error( + load_json, respx_mock, client, tmp_path +): + """A column the model does not declare must come in every symbol's + answer, as a declared one must: a symbol without it fails the call on + every format, naming that symbol's answer.""" + call = load_json("options_quotes_response_200") + call["brandNew"] = ["x"] * len(call["optionSymbol"]) + respx_mock.get(CALL_URL).respond(json=call, status_code=200) + respx_mock.get(PUT_URL).respond( + json=load_json("options_quotes_response_200"), status_code=200 + ) + + for output_format in ( + OutputFormat.INTERNAL, + OutputFormat.JSON, + OutputFormat.DATAFRAME, + ): + with pytest.raises(ParseError) as failure: + client.options.quotes( + symbols=THREE_SYMBOLS[:2], output_format=output_format + ) + assert failure.value.request_url.startswith(PUT_URL) + assert "missing columns ['brandNew']" in failure.value.message + respx_mock.get(CALL_URL).respond(text=f"{CSV_HEADER},brandNew\n{CALL_ROW},x\n") + respx_mock.get(PUT_URL).respond(text=f"{CSV_HEADER}\n{PUT_ROW}\n") + with pytest.raises(ParseError) as failure: + client.options.quotes( + symbols=THREE_SYMBOLS[:2], + output_format=OutputFormat.CSV, + filename=tmp_path / "test.csv", + ) + assert failure.value.request_url.startswith(PUT_URL) + assert "differs from" in failure.value.message + + def test_options_quotes_csv_keeps_the_requested_columns(respx_mock, client, tmp_path): """Issue #86: under `columns=` the API answers with the requested columns only; the merge used to drop every row because the header did not match diff --git a/src/tests/test_utils.py b/src/tests/test_utils.py index 931c72a..32da326 100644 --- a/src/tests/test_utils.py +++ b/src/tests/test_utils.py @@ -332,14 +332,16 @@ def test_merge_csv_responses_reports_an_unreadable_body_before_a_misaligned_row( def test_json_answer_columns_follow_the_order_the_answers_send(): """Under `columns=` the API sends the requested keys only, in request - order (checked live: `columns=v,c` answers `{"v": [...], "c": [...]}`). - The merge keeps that order, which is the order of the empty result and of - every single-request resource, not the model's; the status flag and keys - the model does not know are not columns.""" + order, and the merge keeps that order, not the model's. A key the model + does not declare is a column too when it holds a list; the status flag and + a key that holds one value are not.""" responses = [_csv_response(""), _csv_response("")] - answers = [{"c": [1], "t": [2], "x": [0]}, {"c": [4], "t": [3], "s": "ok"}] + answers = [ + {"c": [1], "t": [2], "x": [0], "note": "one value"}, + {"c": [4], "t": [3], "x": [5], "s": "ok"}, + ] - assert json_answer_columns(responses, answers, COLUMNS) == ["c", "t"] + assert json_answer_columns(responses, answers, COLUMNS) == ["c", "t", "x"] @pytest.mark.parametrize( @@ -358,6 +360,7 @@ def test_json_answer_columns_follow_the_order_the_answers_send(): ([{"t": [1], "c": [2]}, {"t": [3], "c": []}], "different lengths", 1), ([{"t": [1], "c": [2]}, {"t": [3], "c": "4.5"}], "are not lists", 1), ([{"t": [1], "c": [2]}, {"t": [3], "c": None}], "are not lists", 1), + ([{"t": [1], "x": [0]}, {"t": [3]}], "missing columns ['x']", 1), ], ids=[ "null", @@ -369,6 +372,7 @@ def test_json_answer_columns_follow_the_order_the_answers_send(): "an-empty-column", "a-column-that-is-a-string", "a-null-column", + "an-undeclared-column-one-answer-lacks", ], ) def test_json_answer_columns_name_the_answer_that_breaks_the_merge( From 7f929dc24902e4c3af97f99d93a8a95f109b2aa8 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 14:34:09 -0300 Subject: [PATCH 08/13] docs(readme): document get_extra --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 9af93ee..6016343 100644 --- a/README.md +++ b/README.md @@ -353,6 +353,27 @@ Mixing `Decimal` and `float` in arithmetic raises `TypeError`, and comparing the The other formats keep the plain parse. A DataFrame never holds a `Decimal`: it keeps the plain parse, so every column has the dtype it always had on both pandas and polars (`float64` for prices with a fraction). A DataFrame is for vectorized analysis, pandas has no decimal dtype, and the same call returning a different dtype depending on which library is installed would be a trap. `OutputFormat.JSON` returns the decoded JSON with standard `float` numbers, and `OutputFormat.CSV` writes the API's text as it came (`client.utilities` builds its CSV from the decoded body). A `NaN` or an `Infinity` in a JSON body is not JSON and fails the call on every format that decodes the body; a number past what a float holds (`1e400`) is `inf` on `JSON` and `DATAFRAME` and exact on `INTERNAL`. +### Columns a model does not declare + +The API adds columns without an SDK release. A model is built from the columns it declares, and `marketdata.get_extra()` returns the others, named as the API sent them, until a release declares them: + +```python +import marketdata +from marketdata import MarketDataClient, OutputFormat + +client = MarketDataClient() +chain = client.options.chain("AAPL", output_format=OutputFormat.INTERNAL) +marketdata.get_extra(chain) # {} while the model declares every column + +quotes = client.stocks.quotes("AAPL", output_format=OutputFormat.INTERNAL) +marketdata.get_extra(quotes[0]) # the row's value of each column StockQuote lacks +``` + +- **Which results:** every `OutputFormat.INTERNAL` result built from an answer's columns, the merged answers of `stocks.candles()` and `options.quotes()` included, and a model built with `from_dict()` on `StockEarnings`, `StockPrice`, `StockQuote` and their human-readable twins. A single-object result (`options.chain()`, `options.expirations()`, `options.lookup()`, `options.quotes()`, `stocks.earnings()`) holds each column whole; a list result (`stocks.quotes()`, `stocks.prices()`, `stocks.candles()`, `stocks.news()`, `funds.candles()`, `markets.status()`, `utilities.status()`) gives each row its own value. `utilities.user()` and `utilities.headers()` keep nothing there: the first reads only its own keys, the second keeps every key as a header. +- **The values:** decoded as the rest of the answer, so a number with a fraction is a `Decimal` on the resources whose money is. Each answer that brings such a column names it once at DEBUG on `marketdata.logger`. +- **How long they last:** as long as the object, which holds them itself, like the response metadata: they survive `copy.copy()`, `copy.deepcopy()` and `pickle`. They are not fields, so `==`, the repr and `dataclasses.fields()` ignore them, and a model built by hand, or from an answer without such a column, returns `{}`. +- **The other formats:** `JSON` and `DATAFRAME` carry the column by the API's name and `CSV` writes it in the file. On `stocks.candles()` and `options.quotes()` every chunk or symbol must carry it, as it must carry a declared column: an answer without it fails the call with `ParseError`, on every format. + ## Universal Parameters All resource methods support universal parameters that can be used to customize the API request and response: From 77592486851516d41c4cc6ac8bb63aa69d898a69 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 14:56:50 -0300 Subject: [PATCH 09/13] feat(resources)!: refuse a foreign object on every format of the single-object resources options.chain, options.expirations, options.lookup and stocks.earnings decode their INTERNAL, JSON and DATAFRAME answers through one check that now also requires at least one of the model's columns, the rule the fan-out merge applies, so an object with none of them, like a proxy's JSON error page, raises the same ParseError on those three formats as its header does on CSV. The missing-column test moves from options.lookup to options.expirations: the lookup model has one column, so removing it leaves an object with none of the model's columns. BREAKING CHANGE: on options.chain, options.expirations, options.lookup and stocks.earnings, JSON and DATAFRAME raise ParseError for an object that carries none of the resource's columns, where they returned it. --- CHANGELOG.md | 2 +- src/marketdata/resources/base.py | 13 +++- src/marketdata/resources/options/chain.py | 6 +- .../resources/options/expirations.py | 6 +- src/marketdata/resources/options/lookup.py | 6 +- src/marketdata/resources/stocks/earnings.py | 6 +- src/tests/test_money_decimal.py | 74 +++++++++++++++---- 7 files changed, 84 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9279385..f31fe5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Money in the `OutputFormat.INTERNAL` models is `decimal.Decimal` instead of `float`, built from the digits in the response body, which those resources now decode with `parse_float=Decimal` on the INTERNAL path: `ask`, `bid`, `mid`, `last`, `change` of `StockQuote`, `mid` and `change` of `StockPrice`, `o`, `h`, `l`, `c` of `StockCandle` and `FundsCandle`, the three EPS lists of `StockEarnings`, `strike`, `bid`, `mid`, `ask`, `last`, `intrinsicValue`, `extrinsicValue`, `underlyingPrice` of `OptionsChain` and `OptionsQuotes`, and the same fields of every human-readable twin. Greeks, IV and percentages stay `float`, sizes and counts `int`; DataFrames (pandas and polars), JSON and CSV output keep the plain float parse. Migration: arithmetic mixing `Decimal` and `float` raises `TypeError`, and a comparison with a float literal compares against its binary value (`Decimal("65.1") == 65.1` is `False`), so use `Decimal` literals or `int`; `float(value)` gives a float back, and `OutputFormat.DATAFRAME` is the float path for analysis (a pandas DataFrame built from models gets `object` columns, and `json.dumps(..., default=str)` writes the money as strings). A model built by hand converts its money arguments: a float or a numpy scalar through its shortest repr (`65.1` is `Decimal("65.1")`), a NaN number, which is how pandas writes a missing value, to `None`; a value that is not a number, a string that reads as NaN, or an infinity raises and names the field, while an array (anything that is not a string and has `__array__` and `__len__`: a numpy array, a pandas or polars Series, a pandas Index) is left as it came, and a `dict`, a `set` or a generator raises. On the API path the same refusal is a `ParseError` naming the field and the body, not a bare `TypeError` (#50) - Dates on `OutputFormat.INTERNAL` are read with the API's own rule and always come back as US/Eastern datetimes. A number from 10000 up to 200000 is a spreadsheet serial of US/Eastern wall-clock time, from 200000 Unix seconds, from `1e10` milliseconds and from `1e13` nanoseconds. A number under 10000 is a relative range for the API, not a date, and raises `ParseError`. The SDK used a boundary of its own at 60000, so from April 2064 a serial would have read as 1970. A `timestamp` date (`2026-09-21`) and a spreadsheet serial are now midnight and wall-clock time in US/Eastern, where they came back as naive datetimes, and a serial is rounded to the second. The intraday `from_date` and `to_date` of `stocks.candles()` are read only when they are ISO dates or numbers the API reads as dates (a spreadsheet serial or a Unix time), which the SDK splits into one request per year: a relative range or a keyword goes to the API as it is, in one request, where `"60"` was sent as 1900-02-28 and `"yesterday"` was refused. `UserRateLimits.reset_time` takes a datetime or Unix seconds, and refuses anything else (#116). Migration: compare INTERNAL dates against aware datetimes (`pytz.timezone("US/Eastern").localize(...)`), not naive ones - A response body carrying a `NaN`, `Infinity` or `-Infinity` literal raises `ParseError` on every output format that decodes it, where `INTERNAL`, `JSON` and `DATAFRAME` used to return it as `nan` or `inf`. Those literals are not JSON, and the renderer behind the API's data endpoints refuses them, answering `500`, so one in their bodies was written by something in between (`/status/` and `/headers/` are served elsewhere). `CSV` output writes the API's body as it came, since it reads no number, except for `client.utilities`, which builds its CSV from the decoded body and refuses it too. A number past what a float holds is not refused: `JSON` and `DATAFRAME` read `1e400` as `inf` while `INTERNAL` keeps it exact, and `INTERNAL` refuses one past what a `Decimal` holds. That renderer does not write such a number, and refusing it on the float path would slow every decode. Migration: code that read `nan` or `inf` from such a body catches `ParseError` (or `BaseMarketdataException`) instead; no setting brings the literals back (#50) -- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`), and their CSV output raises `ParseError` too, since it now refuses a header row with none of the resource's columns and, under `add_headers=False`, a body that reads as JSON or HTML, where it wrote the body to the file. The CSV output of `stocks.candles()` and `options.quotes()` under `add_headers=False` refuses such a body the same way. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) +- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`), and their CSV output raises `ParseError` too, since it now refuses a header row with none of the resource's columns and, under `add_headers=False`, a body that reads as JSON or HTML, where it wrote the body to the file. The CSV output of `stocks.candles()` and `options.quotes()` under `add_headers=False` refuses such a body the same way. On those four resources an object with none of their columns, like a proxy's JSON error page, raises `ParseError` on `JSON` and `DATAFRAME` too, where they returned it, as it does on `INTERNAL` and CSV. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) - `options.chain(delta=...)` checks a number before it sends it, the way it checks `strike`, and keeps it as the text it sends. A `bool`, a NaN, an infinity, a number a float cannot hold and a delta outside -1 to 1 now raise `ValidationError` before any request, where they used to be sent: `delta=nan` came back as the whole chain, since the API reads it as no filter, `delta=True` was sent as `1.0`, and `delta=30` was sent as `30.0`. `OptionsChainInput.delta` holds that text (`"0.5"`) instead of a `float` (#101). Migration: pass a finite number from -1 to 1, the way the public docs write a delta (`0.3`), a `DeltaFilter`, or `delta=None` for no filter; code that reads `OptionsChainInput(...).delta` gets the text sent to the API, and `float()` turns a plain number back. - Every date field of the `OutputFormat.INTERNAL` models admits `None` in its annotation: `datetime.datetime | None` for one date, `list[datetime.datetime | None]` for a column of them, and `datetime.date | None` for `MarketStatus.date`, as `OptionsExpirations.updated` already did. A `null` date in an answer reads as `None` in its own row (see Fixed), and the annotations now say so. Migration: a type checker now asks for a `None` check before a `datetime` method is called on a date field (`if quote.updated is not None: quote.updated.date()`). At run time nothing that used to return changes: an answer with a `null` date used to raise `ParseError`, or on `OptionsQuotes` shift the column (#106) - A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model is built without it, where the call raised a bare `TypeError`, and `marketdata.get_extra(model)` returns it as the API sent it: the whole column on a single-object model, the row's own value on each row of a list result. The column is named once per answer at DEBUG on `marketdata.logger`. `StockEarnings.from_dict()`, `StockPrice.from_dict()`, `StockQuote.from_dict()` and their human-readable twins keep such a key for `get_extra()` the same way, where they raised `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` carry such a column through their merge on every format, INTERNAL through `get_extra()`, as long as every chunk or symbol sends it: like a declared column, an answer without it, or with a list of another length, fails the call with `ParseError` on every format. Their CSV output keeps it, where it raised `ParseError`, because `merge_csv_responses()` now refuses only a header with none of the resource's columns. Migration: code that caught the `TypeError` or `ParseError` an added column raised now gets the result; read the column with `get_extra()` until the model declares it (#111) diff --git a/src/marketdata/resources/base.py b/src/marketdata/resources/base.py index a4699d5..c4aed1c 100644 --- a/src/marketdata/resources/base.py +++ b/src/marketdata/resources/base.py @@ -57,11 +57,15 @@ def model_columns(output_model: type, requested: list[str] | None = None) -> lis return _model_columns(output_model, requested) -def _parse_json_object(response: Response, *, exact: bool = False) -> dict[str, Any]: - """Decode a response body that must be a JSON object. +def _parse_json_object( + response: Response, output_model: type, *, exact: bool = False +) -> dict[str, Any]: + """Decode a response body that must be a JSON object carrying at least one + of the output model's columns. Args: response: The answer to decode. + output_model: The output model of the resource. exact: Decode each number with a fraction as a ``Decimal``, as ``parse_json`` does. @@ -70,11 +74,14 @@ def _parse_json_object(response: Response, *, exact: bool = False) -> dict[str, Raises: ParseError: If the body is not valid JSON, holds a number that is not - finite, or is not a JSON object. + finite, is not a JSON object, or carries none of the model's + columns. """ data = parse_json(response, exact=exact) if not isinstance(data, dict): raise parse_error(response, "not a JSON object") + if not any(column in data for column in model_columns(output_model)): + raise parse_error(response, "none of this resource's fields") return data diff --git a/src/marketdata/resources/options/chain.py b/src/marketdata/resources/options/chain.py index ef6cdb1..719ec6d 100644 --- a/src/marketdata/resources/options/chain.py +++ b/src/marketdata/resources/options/chain.py @@ -67,20 +67,20 @@ def chain( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = _parse_json_object(response) + data = _parse_json_object(response, output_model) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["optionSymbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = _parse_json_object(response, exact=True) + data = _parse_json_object(response, output_model, exact=True) fields, extra = _split_fields(output_model, data) with model_errors(response): return _with_extra(output_model(**fields), extra) elif user_universal_params.output_format == OutputFormat.JSON: - return _parse_json_object(response) + return _parse_json_object(response, output_model) elif user_universal_params.output_format == OutputFormat.CSV: _check_csv_header( diff --git a/src/marketdata/resources/options/expirations.py b/src/marketdata/resources/options/expirations.py index 0b24722..5f91daf 100644 --- a/src/marketdata/resources/options/expirations.py +++ b/src/marketdata/resources/options/expirations.py @@ -79,20 +79,20 @@ def expirations( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = _parse_json_object(response) + data = _parse_json_object(response, output_model) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=index_columns ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = _parse_json_object(response) + data = _parse_json_object(response, output_model) fields, extra = _split_fields(output_model, data) with model_errors(response): return _with_extra(output_model(**fields), extra) elif user_universal_params.output_format == OutputFormat.JSON: - return _parse_json_object(response) + return _parse_json_object(response, output_model) elif user_universal_params.output_format == OutputFormat.CSV: _check_csv_header( diff --git a/src/marketdata/resources/options/lookup.py b/src/marketdata/resources/options/lookup.py index eb805f9..6edf477 100644 --- a/src/marketdata/resources/options/lookup.py +++ b/src/marketdata/resources/options/lookup.py @@ -71,20 +71,20 @@ def lookup( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = _parse_json_object(response) + data = _parse_json_object(response, output_model) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["optionSymbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = _parse_json_object(response) + data = _parse_json_object(response, output_model) fields, extra = _split_fields(output_model, data) with model_errors(response): return _with_extra(output_model(**fields), extra) elif user_universal_params.output_format == OutputFormat.JSON: - return _parse_json_object(response) + return _parse_json_object(response, output_model) elif user_universal_params.output_format == OutputFormat.CSV: _check_csv_header( diff --git a/src/marketdata/resources/stocks/earnings.py b/src/marketdata/resources/stocks/earnings.py index a51bc8f..bb2ed27 100644 --- a/src/marketdata/resources/stocks/earnings.py +++ b/src/marketdata/resources/stocks/earnings.py @@ -66,19 +66,19 @@ def earnings( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = _parse_json_object(response) + data = _parse_json_object(response, output_model) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["symbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = _parse_json_object(response, exact=True) + data = _parse_json_object(response, output_model, exact=True) with model_errors(response): return output_model.from_dict(data) elif user_universal_params.output_format == OutputFormat.JSON: - return _parse_json_object(response) + return _parse_json_object(response, output_model) elif user_universal_params.output_format == OutputFormat.CSV: _check_csv_header( diff --git a/src/tests/test_money_decimal.py b/src/tests/test_money_decimal.py index 9cc0fb7..1112920 100644 --- a/src/tests/test_money_decimal.py +++ b/src/tests/test_money_decimal.py @@ -669,7 +669,9 @@ def test_a_value_the_model_cannot_hold_is_a_parse_error_not_a_builtin( assert body["mid"] == [decoded] -@pytest.mark.parametrize("name", ["options.lookup", "markets.status", "utilities.user"]) +@pytest.mark.parametrize( + "name", ["options.expirations", "markets.status", "utilities.user"] +) def test_a_body_missing_a_column_the_model_needs_is_a_parse_error( respx_mock, client, name ): @@ -837,19 +839,65 @@ def test_an_object_with_none_of_the_model_columns_is_a_parse_error( ) +SINGLE_OBJECT_MODEL_CASES = [ + "options.chain", + "options.chain human", + "options.expirations", + "options.expirations human", + "options.lookup", + "stocks.earnings", + "stocks.earnings human", +] + + +@pytest.mark.parametrize("name", SINGLE_OBJECT_MODEL_CASES) +def test_a_foreign_object_is_a_parse_error_on_every_format( + respx_mock, client, tmp_path, name +): + """A single-object resource refuses a JSON object that carries none of its + columns, like a proxy's JSON error page, on every format: the same + ``ParseError`` on the three that decode it, and on CSV the one a header + with none of the resource's columns gets or, without a header row, the one + a body that reads as JSON gets.""" + case = MODEL_CASES[name] + text = json.dumps({"error": "upstream timeout"}) + _respond(respx_mock, case, text.encode()) + + messages = set() + for output_format in ( + OutputFormat.INTERNAL, + OutputFormat.JSON, + OutputFormat.DATAFRAME, + ): + with pytest.raises(ParseError) as failure: + case.call(client, output_format=output_format) + messages.add(failure.value.message) + with pytest.raises(ParseError) as refused: + case.call( + client, output_format=OutputFormat.CSV, filename=tmp_path / "answer.csv" + ) + with pytest.raises(ParseError) as headerless: + case.call( + client, + output_format=OutputFormat.CSV, + add_headers=False, + filename=tmp_path / "answer.csv", + ) + + assert messages == { + "Response body is not a valid answer of this resource (none of this" + f" resource's fields): {text!r}" + } + assert refused.value.message == ( + "Response body is not a valid answer of this resource (unknown columns" + f" [{text!r}]): {text!r}" + ) + assert "(JSON or HTML, not CSV)" in headerless.value.message + assert not (tmp_path / "answer.csv").exists() + + @pytest.mark.parametrize("body", [b"[1]", b"null"]) -@pytest.mark.parametrize( - "name", - [ - "options.chain", - "options.chain human", - "options.expirations", - "options.expirations human", - "options.lookup", - "stocks.earnings", - "stocks.earnings human", - ], -) +@pytest.mark.parametrize("name", SINGLE_OBJECT_MODEL_CASES) def test_a_body_that_is_not_an_object_is_a_parse_error_on_every_format( respx_mock, client, tmp_path, name, body ): From 1519681aee4abb752fcb1e2fe3a5db574e90b348 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 15:25:09 -0300 Subject: [PATCH 10/13] test(stocks): pin both quote from_dict with a column the model does not declare --- src/tests/test_stocks_quotes.py | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/tests/test_stocks_quotes.py b/src/tests/test_stocks_quotes.py index 6a92011..55dd1f1 100644 --- a/src/tests/test_stocks_quotes.py +++ b/src/tests/test_stocks_quotes.py @@ -7,6 +7,7 @@ import pytest import pytz +import marketdata from marketdata.exceptions import ServerError from marketdata.input_types.base import DateFormat, OutputFormat from marketdata.output_types.stocks_quotes import StockQuote, StockQuotesHumanReadable @@ -181,6 +182,53 @@ def test_stock_quotes_human_readable_from_dict(): ) +def test_stock_quote_from_dict_keeps_an_undeclared_column_for_get_extra(): + """StockQuote.from_dict builds the quote from one row keyed by the API's + names and keeps a column the model does not declare for get_extra.""" + row = { + "symbol": "AAPL", + "ask": 278.02, + "askSize": 100, + "bid": 277.97, + "bidSize": 100, + "mid": 277.995, + "last": 278.0188, + "change": -0.0112, + "changepct": 0.0, + "volume": 4964676, + "updated": 1765478200, + } + + instance = StockQuote.from_dict({**row, "brandNew": 1}) + + assert instance == StockQuote.from_dict(row) + assert marketdata.get_extra(instance) == {"brandNew": 1} + + +def test_stock_quotes_human_readable_from_dict_keeps_an_undeclared_column(): + """StockQuotesHumanReadable.from_dict builds the quote from one row keyed by + the human-readable names and keeps a column the model does not declare for + get_extra.""" + row = { + "Symbol": "AAPL", + "Ask": 278.02, + "Ask Size": 100, + "Bid": 277.97, + "Bid Size": 100, + "Mid": 277.995, + "Last": 278.0188, + "Change $": 0.51, + "Change %": 0.0018, + "Volume": 4964676, + "Date": 1765478200, + } + + instance = StockQuotesHumanReadable.from_dict({**row, "brandNew": 1}) + + assert instance == StockQuotesHumanReadable.from_dict(row) + assert marketdata.get_extra(instance) == {"brandNew": 1} + + def test_get_stocks_quotes_response_200_internal(load_json, respx_mock, client): mock_data = load_json("stocks_quotes_response_200") updated = datetime.datetime.fromtimestamp( From 21efa027a5f108d6763e8cd633ac6a279bed7840 Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Mon, 28 Sep 2026 15:25:09 -0300 Subject: [PATCH 11/13] docs: document get_extra on each resource page --- docs/funds.md | 4 ++++ docs/markets.md | 4 ++++ docs/options.md | 4 ++++ docs/stocks.md | 4 ++++ docs/utilities.md | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/docs/funds.md b/docs/funds.md index 10e57d4..6e1cde9 100644 --- a/docs/funds.md +++ b/docs/funds.md @@ -13,6 +13,10 @@ funds = client.funds All methods in the funds resource include API status checking and automatic retry logic. See the [README](../README.md) for general information about error handling, retry mechanisms, and output formats. +## Columns a model does not declare + +With `OutputFormat.INTERNAL`, a column the API sends before the SDK declares it does not fail the call: the result is built from the columns its model declares, and `marketdata.get_extra(candle)` returns the others, named as the API sent them, with each row's own value. `JSON` and `DATAFRAME` carry it as a column, and `CSV` writes it in the file. See [Columns a model does not declare](../README.md#columns-a-model-does-not-declare) in the README for how long the values last and how they are decoded. + ## Methods ### `candles()` diff --git a/docs/markets.md b/docs/markets.md index e99ceb6..787089b 100644 --- a/docs/markets.md +++ b/docs/markets.md @@ -13,6 +13,10 @@ markets = client.markets All methods in the markets resource include API status checking and automatic retry logic. See the [README](../README.md) for general information about error handling, retry mechanisms, and output formats. +## Columns a model does not declare + +With `OutputFormat.INTERNAL`, a column the API sends before the SDK declares it does not fail the call: the result is built from the columns its model declares, and `marketdata.get_extra(row)` returns the others, named as the API sent them, with each row's own value. `JSON` and `DATAFRAME` carry it as a column, and `CSV` writes it in the file. See [Columns a model does not declare](../README.md#columns-a-model-does-not-declare) in the README for how long the values last and how they are decoded. + ## Methods ### `status()` diff --git a/docs/options.md b/docs/options.md index c16f1eb..648cd37 100644 --- a/docs/options.md +++ b/docs/options.md @@ -13,6 +13,10 @@ options = client.options All methods in the options resource include API status checking and automatic retry logic. See the [README](../README.md) for general information about error handling, retry mechanisms, and output formats. +## Columns a model does not declare + +With `OutputFormat.INTERNAL`, a column the API sends before the SDK declares it does not fail the call: the result is built from the columns its model declares, and `marketdata.get_extra(result)` returns the others, named as the API sent them, each column whole, since `chain()`, `expirations()`, `lookup()` and `quotes()` each return one object. On `quotes()` every symbol must send such a column, as it must send a declared one, or the call raises `ParseError`. `JSON` and `DATAFRAME` carry it as a column, and `CSV` writes it in the file. See [Columns a model does not declare](../README.md#columns-a-model-does-not-declare) in the README for how long the values last and how they are decoded. + ## Methods ### `expirations()` diff --git a/docs/stocks.md b/docs/stocks.md index 19cc9ad..62901fa 100644 --- a/docs/stocks.md +++ b/docs/stocks.md @@ -13,6 +13,10 @@ stocks = client.stocks All methods in the stocks resource include API status checking and automatic retry logic. See the [README](../README.md) for general information about error handling, retry mechanisms, and output formats. +## Columns a model does not declare + +With `OutputFormat.INTERNAL`, a column the API sends before the SDK declares it does not fail the call: the result is built from the columns its model declares, and `marketdata.get_extra()` returns the others, named as the API sent them: on each row of `prices()`, `quotes()`, `candles()` and `news()` the row's own value, and on `earnings()`, which returns one object, the whole column. On `candles()` every chunk must send such a column, as it must send a declared one, or the call raises `ParseError`. The `from_dict()` of `StockEarnings`, `StockPrice`, `StockQuote` and their human-readable twins keep such a key the same way. `JSON` and `DATAFRAME` carry it as a column, and `CSV` writes it in the file. See [Columns a model does not declare](../README.md#columns-a-model-does-not-declare) in the README for how long the values last and how they are decoded. + ## Methods ### `prices()` diff --git a/docs/utilities.md b/docs/utilities.md index df61a36..4df109c 100644 --- a/docs/utilities.md +++ b/docs/utilities.md @@ -15,6 +15,10 @@ All three methods include automatic retry logic. They never consume API credits, These endpoints live at the API root (`/status/`, `/headers/`, `/user/`), only speak JSON and accept no query parameters. The SDK therefore sends them bare and produces every output format client-side from the decoded JSON. Of the universal parameters, only `output_format` and `filename` apply. +## Columns a model does not declare + +With `OutputFormat.INTERNAL`, a column the API sends before the SDK declares it does not fail the call: the result is built from the columns its model declares, and `marketdata.get_extra(row)` returns the others on each row of `status()`, named as the API sent them, with the row's own value. `user()` reads only its own keys and `headers()` keeps every key as a header, so neither keeps anything there. `JSON` and `DATAFRAME` carry it as a column, and `CSV` writes it in the file. See [Columns a model does not declare](../README.md#columns-a-model-does-not-declare) in the README for how long the values last and how they are decoded. + ## Methods ### `status()` From 4fb87ed9b15fbc01b4533fb87e3c8be6bf480c9a Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Fri, 2 Oct 2026 12:41:47 -0300 Subject: [PATCH 12/13] feat(resources)!: refuse a headerless CSV answer labeled JSON or HTML Under add_headers=False there is no header row to check, and a lone string, number or boolean reads as a one-cell CSV: the API's own headerless answers include 1790740800, which is valid JSON too. The API labels every CSV answer text/csv and every JSON one application/json, so the CSV output of options.chain, options.expirations, options.lookup, stocks.earnings, stocks.candles and options.quotes now refuses an answer whose Content-Type names JSON or HTML, whatever its body, and keeps the check on the body's shape under any other label or none. The tests label their CSV answers text/csv, as the API does. BREAKING CHANGE: under add_headers=False, the CSV output of options.chain, options.expirations, options.lookup, stocks.earnings, stocks.candles and options.quotes raises ParseError for an answer whose Content-Type names JSON or HTML, where it wrote the body to the file. --- CHANGELOG.md | 2 +- src/marketdata/resources/base.py | 5 +- src/marketdata/utils.py | 31 +++++++---- src/tests/test_money_decimal.py | 91 +++++++++++++++++++++++--------- 4 files changed, 90 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f31fe5e..1c910b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Money in the `OutputFormat.INTERNAL` models is `decimal.Decimal` instead of `float`, built from the digits in the response body, which those resources now decode with `parse_float=Decimal` on the INTERNAL path: `ask`, `bid`, `mid`, `last`, `change` of `StockQuote`, `mid` and `change` of `StockPrice`, `o`, `h`, `l`, `c` of `StockCandle` and `FundsCandle`, the three EPS lists of `StockEarnings`, `strike`, `bid`, `mid`, `ask`, `last`, `intrinsicValue`, `extrinsicValue`, `underlyingPrice` of `OptionsChain` and `OptionsQuotes`, and the same fields of every human-readable twin. Greeks, IV and percentages stay `float`, sizes and counts `int`; DataFrames (pandas and polars), JSON and CSV output keep the plain float parse. Migration: arithmetic mixing `Decimal` and `float` raises `TypeError`, and a comparison with a float literal compares against its binary value (`Decimal("65.1") == 65.1` is `False`), so use `Decimal` literals or `int`; `float(value)` gives a float back, and `OutputFormat.DATAFRAME` is the float path for analysis (a pandas DataFrame built from models gets `object` columns, and `json.dumps(..., default=str)` writes the money as strings). A model built by hand converts its money arguments: a float or a numpy scalar through its shortest repr (`65.1` is `Decimal("65.1")`), a NaN number, which is how pandas writes a missing value, to `None`; a value that is not a number, a string that reads as NaN, or an infinity raises and names the field, while an array (anything that is not a string and has `__array__` and `__len__`: a numpy array, a pandas or polars Series, a pandas Index) is left as it came, and a `dict`, a `set` or a generator raises. On the API path the same refusal is a `ParseError` naming the field and the body, not a bare `TypeError` (#50) - Dates on `OutputFormat.INTERNAL` are read with the API's own rule and always come back as US/Eastern datetimes. A number from 10000 up to 200000 is a spreadsheet serial of US/Eastern wall-clock time, from 200000 Unix seconds, from `1e10` milliseconds and from `1e13` nanoseconds. A number under 10000 is a relative range for the API, not a date, and raises `ParseError`. The SDK used a boundary of its own at 60000, so from April 2064 a serial would have read as 1970. A `timestamp` date (`2026-09-21`) and a spreadsheet serial are now midnight and wall-clock time in US/Eastern, where they came back as naive datetimes, and a serial is rounded to the second. The intraday `from_date` and `to_date` of `stocks.candles()` are read only when they are ISO dates or numbers the API reads as dates (a spreadsheet serial or a Unix time), which the SDK splits into one request per year: a relative range or a keyword goes to the API as it is, in one request, where `"60"` was sent as 1900-02-28 and `"yesterday"` was refused. `UserRateLimits.reset_time` takes a datetime or Unix seconds, and refuses anything else (#116). Migration: compare INTERNAL dates against aware datetimes (`pytz.timezone("US/Eastern").localize(...)`), not naive ones - A response body carrying a `NaN`, `Infinity` or `-Infinity` literal raises `ParseError` on every output format that decodes it, where `INTERNAL`, `JSON` and `DATAFRAME` used to return it as `nan` or `inf`. Those literals are not JSON, and the renderer behind the API's data endpoints refuses them, answering `500`, so one in their bodies was written by something in between (`/status/` and `/headers/` are served elsewhere). `CSV` output writes the API's body as it came, since it reads no number, except for `client.utilities`, which builds its CSV from the decoded body and refuses it too. A number past what a float holds is not refused: `JSON` and `DATAFRAME` read `1e400` as `inf` while `INTERNAL` keeps it exact, and `INTERNAL` refuses one past what a `Decimal` holds. That renderer does not write such a number, and refusing it on the float path would slow every decode. Migration: code that read `nan` or `inf` from such a body catches `ParseError` (or `BaseMarketdataException`) instead; no setting brings the literals back (#50) -- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`), and their CSV output raises `ParseError` too, since it now refuses a header row with none of the resource's columns and, under `add_headers=False`, a body that reads as JSON or HTML, where it wrote the body to the file. The CSV output of `stocks.candles()` and `options.quotes()` under `add_headers=False` refuses such a body the same way. On those four resources an object with none of their columns, like a proxy's JSON error page, raises `ParseError` on `JSON` and `DATAFRAME` too, where they returned it, as it does on `INTERNAL` and CSV. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) +- With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`), and their CSV output raises `ParseError` too, since it now refuses a header row with none of the resource's columns and, under `add_headers=False`, an answer whose `Content-Type` names JSON or HTML, a lone string, number or boolean included, or whose body reads as either, where it wrote the body to the file. The CSV output of `stocks.candles()` and `options.quotes()` under `add_headers=False` refuses such an answer the same way. On those four resources an object with none of their columns, like a proxy's JSON error page, raises `ParseError` on `JSON` and `DATAFRAME` too, where they returned it, as it does on `INTERNAL` and CSV. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) - `options.chain(delta=...)` checks a number before it sends it, the way it checks `strike`, and keeps it as the text it sends. A `bool`, a NaN, an infinity, a number a float cannot hold and a delta outside -1 to 1 now raise `ValidationError` before any request, where they used to be sent: `delta=nan` came back as the whole chain, since the API reads it as no filter, `delta=True` was sent as `1.0`, and `delta=30` was sent as `30.0`. `OptionsChainInput.delta` holds that text (`"0.5"`) instead of a `float` (#101). Migration: pass a finite number from -1 to 1, the way the public docs write a delta (`0.3`), a `DeltaFilter`, or `delta=None` for no filter; code that reads `OptionsChainInput(...).delta` gets the text sent to the API, and `float()` turns a plain number back. - Every date field of the `OutputFormat.INTERNAL` models admits `None` in its annotation: `datetime.datetime | None` for one date, `list[datetime.datetime | None]` for a column of them, and `datetime.date | None` for `MarketStatus.date`, as `OptionsExpirations.updated` already did. A `null` date in an answer reads as `None` in its own row (see Fixed), and the annotations now say so. Migration: a type checker now asks for a `None` check before a `datetime` method is called on a date field (`if quote.updated is not None: quote.updated.date()`). At run time nothing that used to return changes: an answer with a `null` date used to raise `ParseError`, or on `OptionsQuotes` shift the column (#106) - A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model is built without it, where the call raised a bare `TypeError`, and `marketdata.get_extra(model)` returns it as the API sent it: the whole column on a single-object model, the row's own value on each row of a list result. The column is named once per answer at DEBUG on `marketdata.logger`. `StockEarnings.from_dict()`, `StockPrice.from_dict()`, `StockQuote.from_dict()` and their human-readable twins keep such a key for `get_extra()` the same way, where they raised `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` carry such a column through their merge on every format, INTERNAL through `get_extra()`, as long as every chunk or symbol sends it: like a declared column, an answer without it, or with a list of another length, fails the call with `ParseError` on every format. Their CSV output keeps it, where it raised `ParseError`, because `merge_csv_responses()` now refuses only a header with none of the resource's columns. Migration: code that caught the `TypeError` or `ParseError` an added column raised now gets the result; read the column with `get_extra()` until the model declares it (#111) diff --git a/src/marketdata/resources/base.py b/src/marketdata/resources/base.py index c4aed1c..b0c5b75 100644 --- a/src/marketdata/resources/base.py +++ b/src/marketdata/resources/base.py @@ -98,10 +98,11 @@ def _check_csv_header( Raises: ParseError: If the answer starts with a header row that names none of the model's columns, or with one the ``csv`` module cannot read, - or, without a header row, if it reads as JSON or HTML. + or, without a header row, if it is labeled or shaped as JSON or + HTML. """ if not with_header: - if _reads_as_json_or_html(response.text): + if _reads_as_json_or_html(response): raise parse_error(response, "JSON or HTML, not CSV") return try: diff --git a/src/marketdata/utils.py b/src/marketdata/utils.py index f371b79..0aec481 100644 --- a/src/marketdata/utils.py +++ b/src/marketdata/utils.py @@ -368,8 +368,8 @@ def merge_csv_responses( the same column still merges. A column of the header that matches none of them is kept. with_header: Whether the bodies start with a header row. Without one, - only the row width is checked, against the first row seen, and a - body that reads as JSON or HTML is refused. + only the row width is checked, against the first row seen, and an + answer labeled or shaped as JSON or HTML is refused. Returns: The first answer's header, if any, then every row of every answer, @@ -380,8 +380,9 @@ def merge_csv_responses( ParseError: Naming the offending response, for a body the ``csv`` module cannot read, a body with no header row, a header with none of the resource's columns or with other columns, or in another - order, than the first answer's, a body without a header row that - reads as JSON or HTML, or a row not as wide as the header. + order, than the first answer's, an answer without a header row + labeled or shaped as JSON or HTML, or a row not as wide as the + header. """ known = {column_key(name) for name in known_columns} header: list[str] | None = None @@ -391,7 +392,7 @@ def merge_csv_responses( writer = csv.writer(output) for response in responses: - if not with_header and _reads_as_json_or_html(response.text): + if not with_header and _reads_as_json_or_html(response): raise parse_error(response, "JSON or HTML, not CSV") rows = _csv_records(response.text) try: @@ -444,17 +445,27 @@ def _unreadable_csv(response: Response, error: csv.Error) -> ParseError: return parse_error(response, f"unreadable CSV: {error}") -def _reads_as_json_or_html(text: str) -> bool: +def _reads_as_json_or_html(response: Response) -> bool: """Tell whether a CSV answer is a JSON value or an HTML page instead. Args: - text: The body. + response: The answer. Returns: - Whether, past a BOM and white space, it starts with ``[``, ``{`` or - ``<``, or is ``null``. + Whether its ``Content-Type`` names JSON or HTML or, under any other + type or none, its body, past a BOM and white space, starts with + ``[``, ``{`` or ``<``, or is ``null``. """ - body = text.lstrip(BOM).strip() + # A one-cell CSV can be valid JSON (1790740800), so only the label tells a + # JSON scalar apart. + media_type = response.headers.get("content-type", "").partition(";")[0] + media_type = media_type.strip().lower() + if media_type.endswith(("/json", "+json")) or media_type in ( + "text/html", + "application/xhtml+xml", + ): + return True + body = response.text.lstrip(BOM).strip() return body[:1] in ("[", "{", "<") or body == "null" diff --git a/src/tests/test_money_decimal.py b/src/tests/test_money_decimal.py index 1112920..92f6312 100644 --- a/src/tests/test_money_decimal.py +++ b/src/tests/test_money_decimal.py @@ -333,12 +333,24 @@ def _load_fixture(name: str) -> dict: return json.loads(path.read_text(encoding="utf-8")) -def _respond(respx_mock, case: Case, body: bytes | None = None) -> None: +def _respond( + respx_mock, + case: Case, + body: bytes | None = None, + content_type: str | None = "application/json", +) -> None: + """Answer every request to a case's URL with one body. + + Args: + respx_mock: The router. + case: The resource. + body: The body; the case's fixture as JSON when omitted. + content_type: The ``Content-Type`` header, or ``None`` for none. + """ if body is None: body = json.dumps(_load_fixture(case.fixture)).encode() - respx_mock.get(case.url).respond( - content=body, headers={"content-type": "application/json"} - ) + headers = {"content-type": content_type} if content_type else {} + respx_mock.get(case.url).respond(content=body, headers=headers) def _holds_decimal(value) -> bool: @@ -896,16 +908,16 @@ def test_a_foreign_object_is_a_parse_error_on_every_format( assert not (tmp_path / "answer.csv").exists() -@pytest.mark.parametrize("body", [b"[1]", b"null"]) +@pytest.mark.parametrize("body", [b"[1]", b"null", b'"x"', b"5", b"true"]) @pytest.mark.parametrize("name", SINGLE_OBJECT_MODEL_CASES) def test_a_body_that_is_not_an_object_is_a_parse_error_on_every_format( respx_mock, client, tmp_path, name, body ): - """A single-object resource refuses a body that is not a JSON object on + """A single-object resource refuses a JSON answer that is not an object on every format, under the API's names and the human-readable ones: the same ``ParseError`` on the three that decode it, and on CSV the one a header with none of the resource's columns gets or, without a header row, the one - a body that reads as JSON gets.""" + a JSON answer gets.""" case = MODEL_CASES[name] _respond(respx_mock, case, body) @@ -931,13 +943,14 @@ def test_a_body_that_is_not_an_object_is_a_parse_error_on_every_format( ) text = body.decode() + header = next(csv.reader([text])) assert messages == { "Response body is not a valid answer of this resource (not a JSON object):" f" {text!r}" } assert refused.value.message == ( "Response body is not a valid answer of this resource (unknown columns" - f" [{text!r}]): {text!r}" + f" {header!r}): {text!r}" ) assert headerless.value.message == ( "Response body is not a valid answer of this resource (JSON or HTML, not" @@ -952,17 +965,21 @@ def test_a_body_that_is_not_an_object_is_a_parse_error_on_every_format( "options.lookup", "stocks.earnings", ] +FAN_OUT_CASES = ["stocks.candles", "options.quotes"] -@pytest.mark.parametrize("name", SINGLE_OBJECT_CASES) +@pytest.mark.parametrize( + "body", [b"AAPL,200\r\n", b"1790740800\r\n"], ids=["a row", "one cell"] +) +@pytest.mark.parametrize("name", SINGLE_OBJECT_CASES + FAN_OUT_CASES) def test_a_csv_without_a_header_row_is_written_as_it_came( - respx_mock, client, tmp_path, name + respx_mock, client, tmp_path, name, body ): - """Under ``add_headers=False`` the answer has no header row to check, so - the file is the API's text as it came.""" + """Under ``add_headers=False`` an answer labeled CSV has no header row to + check, so the file is the API's text as it came, a lone number that also + reads as JSON included.""" case = MODEL_CASES[name] - body = b"AAPL,200\r\n" - _respond(respx_mock, case, body) + _respond(respx_mock, case, body, "text/csv; charset=utf-8") path = case.call( client, @@ -975,17 +992,28 @@ def test_a_csv_without_a_header_row_is_written_as_it_came( @pytest.mark.parametrize( - "body", - [b'{"error": "upstream timeout"}', b"502 Bad Gateway"], - ids=["a JSON object", "an HTML page"], + ("body", "content_type"), + [ + (b'{"error": "upstream timeout"}', None), + (b"502 Bad Gateway", "text/plain; charset=utf-8"), + (b"502 Bad Gateway", "text/html; charset=utf-8"), + (b'"x"', "application/problem+json"), + ], + ids=[ + "a JSON object, unlabeled", + "an HTML page labeled text", + "text labeled HTML", + "a string labeled problem+json", + ], ) def test_a_headerless_csv_that_reads_as_json_or_html_is_a_parse_error( - respx_mock, client, tmp_path, body + respx_mock, client, tmp_path, body, content_type ): - """Under ``add_headers=False`` a body that reads as JSON or HTML is not a - CSV answer, so it is refused and no file is written.""" + """Under ``add_headers=False`` an answer labeled JSON or HTML, or whose + body reads as either under another label or none, is not a CSV answer, so + it is refused and no file is written.""" case = MODEL_CASES["options.chain"] - _respond(respx_mock, case, body) + _respond(respx_mock, case, body, content_type) with pytest.raises(ParseError) as refused: case.call( @@ -999,14 +1027,25 @@ def test_a_headerless_csv_that_reads_as_json_or_html_is_a_parse_error( assert not (tmp_path / "answer.csv").exists() -@pytest.mark.parametrize("name", ["stocks.candles", "options.quotes"]) +@pytest.mark.parametrize( + ("body", "content_type"), + [ + (b"[1]", None), + (b'"x"', "application/json"), + (b"5", "application/json"), + (b"true", "application/json"), + ], + ids=["a list, unlabeled", "a string", "a number", "a boolean"], +) +@pytest.mark.parametrize("name", FAN_OUT_CASES) def test_a_fan_out_headerless_csv_that_reads_as_json_is_a_parse_error( - respx_mock, client, tmp_path, name + respx_mock, client, tmp_path, name, body, content_type ): - """The fan-outs' merge refuses a headerless body that reads as JSON, as - their JSON output refuses a body that is not an object.""" + """The fan-outs' merge refuses a headerless answer labeled JSON, or whose + body reads as JSON under no label, as their JSON output refuses a body + that is not an object.""" case = MODEL_CASES[name] - _respond(respx_mock, case, b"[1]") + _respond(respx_mock, case, body, content_type) with pytest.raises(ParseError) as refused: case.call( From a6cd05e59e5142c19777c2aaf51649f398c3114b Mon Sep 17 00:00:00 2001 From: MarketDataDev02 Date: Fri, 2 Oct 2026 15:21:01 -0300 Subject: [PATCH 13/13] feat(resources)!: refuse an answer whose columns differ in length The fan-outs already refused an answer whose lists differ in length, on every format. Every other resource that answers in columns now holds its answer to the same rule, with the same message, on INTERNAL, JSON and DATAFRAME: funds.candles, markets.status, stocks.news, stocks.prices, stocks.quotes and utilities.status, whose CSV is written from the decoded answer and refuses it too, and options.chain, options.expirations, options.lookup and stocks.earnings. Each format read such an answer its own way: INTERNAL gave every row the whole of an added list, or built rows up to the shortest declared column, DATAFRAME padded the short columns with nulls into extra rows, and JSON returned the body. A single value instead of a list is still kept as sent. json_answer_columns and the single-request resources share _check_column_lengths. BREAKING CHANGE: on funds.candles, markets.status, options.chain, options.expirations, options.lookup, stocks.earnings, stocks.news, stocks.prices, stocks.quotes and utilities.status, an answer whose lists differ in length raises ParseError on INTERNAL, JSON and DATAFRAME, and on the CSV of utilities.status, where they returned a result. --- CHANGELOG.md | 2 +- README.md | 1 + src/marketdata/resources/base.py | 30 +++++- src/marketdata/resources/funds/candles.py | 14 ++- src/marketdata/resources/markets/status.py | 15 ++- src/marketdata/resources/stocks/news.py | 14 ++- src/marketdata/resources/stocks/prices.py | 12 ++- src/marketdata/resources/stocks/quotes.py | 17 +++- src/marketdata/resources/utilities/base.py | 13 ++- src/marketdata/utils.py | 31 +++++- src/tests/test_money_decimal.py | 112 ++++++++++++++++++--- 11 files changed, 214 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a13f32e..dfc9da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - With `OutputFormat.INTERNAL`, a body its model refuses raises `ParseError` on every resource, `markets.status()`, `options.expirations()`, `options.lookup()`, `stocks.news()` and `client.utilities` included: a date it cannot parse, which used to escape as a bare `ValueError`, and a column it needs that the body lacks, which used to escape as a bare `TypeError`. For `stocks.candles()` and `options.quotes()` the error names the chunk or the symbol that sent the value, with that answer's own reason. A body that is not a JSON object (a list, a string, a number or `null`) now raises `ParseError`, where it raised a bare `TypeError` or `AttributeError`, on `stocks.earnings()`, `options.lookup()`, `options.chain()` and `options.expirations()`, with the API's names and the human-readable ones. Their `JSON` and `DATAFRAME` output raise the same `ParseError`, where they returned the body and a frame built from it (an empty one for `null`), and their CSV output raises `ParseError` too, since it now refuses a header row with none of the resource's columns and, under `add_headers=False`, an answer whose `Content-Type` names JSON or HTML, a lone string, number or boolean included, or whose body reads as either, where it wrote the body to the file. The CSV output of `stocks.candles()` and `options.quotes()` under `add_headers=False` refuses such an answer the same way. On those four resources an object with none of their columns, like a proxy's JSON error page, raises `ParseError` on `JSON` and `DATAFRAME` too, where they returned it, as it does on `INTERNAL` and CSV. Elsewhere `INTERNAL` raises what it raised before, a bare built-in except on `stocks.candles()` and `options.quotes()`. An empty object still fails with a bare `ValueError`, before any model reads it, on `stocks.quotes()`, `stocks.prices()`, `stocks.news()`, `funds.candles()`, `markets.status()` and `client.utilities.status()` (#110 has the table). Migration: an `except ValueError` or `except TypeError` around those calls no longer catches these errors; catch `ParseError` (or `BaseMarketdataException`) instead (#50) - `options.chain(delta=...)` checks a number before it sends it, the way it checks `strike`, and keeps it as the text it sends. A `bool`, a NaN, an infinity, a number a float cannot hold and a delta outside -1 to 1 now raise `ValidationError` before any request, where they used to be sent: `delta=nan` came back as the whole chain, since the API reads it as no filter, `delta=True` was sent as `1.0`, and `delta=30` was sent as `30.0`. `OptionsChainInput.delta` holds that text (`"0.5"`) instead of a `float` (#101). Migration: pass a finite number from -1 to 1, the way the public docs write a delta (`0.3`), a `DeltaFilter`, or `delta=None` for no filter; code that reads `OptionsChainInput(...).delta` gets the text sent to the API, and `float()` turns a plain number back. - Every date field of the `OutputFormat.INTERNAL` models admits `None` in its annotation: `datetime.datetime | None` for one date, `list[datetime.datetime | None]` for a column of them, and `datetime.date | None` for `MarketStatus.date`, as `OptionsExpirations.updated` already did. A `null` date in an answer reads as `None` in its own row (see Fixed), and the annotations now say so. Migration: a type checker now asks for a `None` check before a `datetime` method is called on a date field (`if quote.updated is not None: quote.updated.date()`). At run time nothing that used to return changes: an answer with a `null` date used to raise `ParseError`, or on `OptionsQuotes` shift the column (#106) -- A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model is built without it, where the call raised a bare `TypeError`, and `marketdata.get_extra(model)` returns it as the API sent it: the whole column on a single-object model, the row's own value on each row of a list result. The column is named once per answer at DEBUG on `marketdata.logger`. `StockEarnings.from_dict()`, `StockPrice.from_dict()`, `StockQuote.from_dict()` and their human-readable twins keep such a key for `get_extra()` the same way, where they raised `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` carry such a column through their merge on every format, INTERNAL through `get_extra()`, as long as every chunk or symbol sends it: like a declared column, an answer without it, or with a list of another length, fails the call with `ParseError` on every format. Their CSV output keeps it, where it raised `ParseError`, because `merge_csv_responses()` now refuses only a header with none of the resource's columns. Migration: code that caught the `TypeError` or `ParseError` an added column raised now gets the result; read the column with `get_extra()` until the model declares it (#111) +- A column the API adds before the SDK declares it no longer fails the call. On `OutputFormat.INTERNAL` the model is built without it, where the call raised a bare `TypeError`, and `marketdata.get_extra(model)` returns it as the API sent it: the whole column on a single-object model, the row's own value on each row of a list result. The column is named once per answer at DEBUG on `marketdata.logger`. `StockEarnings.from_dict()`, `StockPrice.from_dict()`, `StockQuote.from_dict()` and their human-readable twins keep such a key for `get_extra()` the same way, where they raised `TypeError`. A column the model needs and the body lacks still raises `ParseError`, and so does an object with none of the model's columns. `stocks.candles()` and `options.quotes()` carry such a column through their merge on every format, INTERNAL through `get_extra()`, as long as every chunk or symbol sends it: like a declared column, an answer without it, or with a list of another length, fails the call with `ParseError` on every format. The other resources that answer in columns, `funds.candles()`, `markets.status()`, `options.chain()`, `options.expirations()`, `options.lookup()`, `stocks.earnings()`, `stocks.news()`, `stocks.prices()`, `stocks.quotes()` and `client.utilities.status()`, hold each answer to the same rule: a list, declared or not, longer or shorter than the others raises `ParseError` on `INTERNAL`, `JSON` and `DATAFRAME`, and on the CSV of `client.utilities.status()`, which is written from the decoded answer, where `INTERNAL` built the rows up to the shortest declared column, `DATAFRAME` padded the short columns with nulls and `JSON` returned the body; a single value instead of a list is still kept as sent. The CSV output of `stocks.candles()` and `options.quotes()` keeps an added column, where it raised `ParseError`, because `merge_csv_responses()` now refuses only a header with none of the resource's columns. Migration: code that caught the `TypeError` or `ParseError` an added column raised now gets the result; read the column with `get_extra()` until the model declares it, and catch `ParseError` (or `BaseMarketdataException`) for an answer whose columns differ in length (#111) ### Changed diff --git a/README.md b/README.md index 6016343..990b302 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,7 @@ marketdata.get_extra(quotes[0]) # the row's value of each column StockQuote - **The values:** decoded as the rest of the answer, so a number with a fraction is a `Decimal` on the resources whose money is. Each answer that brings such a column names it once at DEBUG on `marketdata.logger`. - **How long they last:** as long as the object, which holds them itself, like the response metadata: they survive `copy.copy()`, `copy.deepcopy()` and `pickle`. They are not fields, so `==`, the repr and `dataclasses.fields()` ignore them, and a model built by hand, or from an answer without such a column, returns `{}`. - **The other formats:** `JSON` and `DATAFRAME` carry the column by the API's name and `CSV` writes it in the file. On `stocks.candles()` and `options.quotes()` every chunk or symbol must carry it, as it must carry a declared column: an answer without it fails the call with `ParseError`, on every format. +- **One value per row:** like a declared column, an added one that is a list must be as long as the others. A list longer or shorter than the rest fails the call with `ParseError` on `INTERNAL`, `JSON` and `DATAFRAME`, whatever the resource, and on the CSV of `utilities.status()`, which is written from the decoded answer. A single value instead of a list is kept as sent, on every row of a list result; the merges of `stocks.candles()` and `options.quotes()` carry only lists. ## Universal Parameters diff --git a/src/marketdata/resources/base.py b/src/marketdata/resources/base.py index c26d538..cda3206 100644 --- a/src/marketdata/resources/base.py +++ b/src/marketdata/resources/base.py @@ -17,6 +17,7 @@ from marketdata.output_types.columns import _column_names, _model_columns from marketdata.settings import settings from marketdata.utils import ( + _check_column_lengths, _csv_records, _reads_as_json_or_html, _unreadable_csv, @@ -61,7 +62,7 @@ def _parse_json_object( response: Response, output_model: type, *, exact: bool = False ) -> dict[str, Any]: """Decode a response body that must be a JSON object carrying at least one - of the output model's columns. + of the output model's columns, with every column one length. Args: response: The answer to decode. @@ -74,14 +75,37 @@ def _parse_json_object( Raises: ParseError: If the body is not valid JSON, holds a number that is not - finite, is not a JSON object, or carries none of the model's - columns. + finite, is not a JSON object, carries none of the model's columns, + or holds lists of different lengths under two of its keys but the + status flag ``s``. """ data = parse_json(response, exact=exact) if not isinstance(data, dict): raise parse_error(response, "not a JSON object") if not any(column in data for column in model_columns(output_model)): raise parse_error(response, "none of this resource's fields") + _check_column_lengths(response, data) + return data + + +def _parse_json_columns(response: Response, *, exact: bool = False) -> Any: + """Decode a column-oriented answer whose columns must all be one length. + + Args: + response: The answer to decode. + exact: Decode each number with a fraction as a ``Decimal``, as + ``parse_json`` does. + + Returns: + The decoded body. + + Raises: + ParseError: If the body is not valid JSON, holds a number that is not + finite, or holds lists of different lengths under two of its keys + but the status flag ``s``. + """ + data = parse_json(response, exact=exact) + _check_column_lengths(response, data) return data diff --git a/src/marketdata/resources/funds/candles.py b/src/marketdata/resources/funds/candles.py index 5353a58..39ae274 100644 --- a/src/marketdata/resources/funds/candles.py +++ b/src/marketdata/resources/funds/candles.py @@ -8,12 +8,16 @@ from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.funds_candles import FundsCandle, FundsCandlesHumanReadable from marketdata.params import universal_params -from marketdata.resources.base import BaseResource, model_errors, no_data_result +from marketdata.resources.base import ( + BaseResource, + _parse_json_columns, + model_errors, + no_data_result, +) from marketdata.utils import ( encode_path_segment, get_data_records, is_no_data, - parse_json, ) @@ -62,14 +66,14 @@ def candles( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_columns(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["t", "Date"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = parse_json(response, exact=True) + data = _parse_json_columns(response, exact=True) fields, extra = _split_fields(output_model, data) data = get_data_records(fields, exclude_keys=["s"]) @@ -77,7 +81,7 @@ def candles( return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_columns(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/marketdata/resources/markets/status.py b/src/marketdata/resources/markets/status.py index 49c97fc..a39180a 100644 --- a/src/marketdata/resources/markets/status.py +++ b/src/marketdata/resources/markets/status.py @@ -11,8 +11,13 @@ MarketStatusHumanReadable, ) from marketdata.params import universal_params -from marketdata.resources.base import BaseResource, model_errors, no_data_result -from marketdata.utils import get_data_records, is_no_data, parse_json +from marketdata.resources.base import ( + BaseResource, + _parse_json_columns, + model_errors, + no_data_result, +) +from marketdata.utils import get_data_records, is_no_data @api_error_handler(service="/v1/markets/status/") @@ -59,21 +64,21 @@ def status( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_columns(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["Date", "date"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - data = parse_json(response) + data = _parse_json_columns(response) fields, extra = _split_fields(output_model, data) data = get_data_records(fields, exclude_keys=["s"]) with model_errors(response): return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_columns(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/marketdata/resources/stocks/news.py b/src/marketdata/resources/stocks/news.py index 4af3b2a..04300e4 100644 --- a/src/marketdata/resources/stocks/news.py +++ b/src/marketdata/resources/stocks/news.py @@ -8,12 +8,16 @@ from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.stocks_news import StockNews, StockNewsHumanReadable from marketdata.params import universal_params -from marketdata.resources.base import BaseResource, model_errors, no_data_result +from marketdata.resources.base import ( + BaseResource, + _parse_json_columns, + model_errors, + no_data_result, +) from marketdata.utils import ( encode_path_segment, get_data_records, is_no_data, - parse_json, ) @@ -62,20 +66,20 @@ def news( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_columns(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["symbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - fields, extra = _split_fields(output_model, parse_json(response)) + fields, extra = _split_fields(output_model, _parse_json_columns(response)) data = get_data_records(fields, exclude_keys=["s"]) with model_errors(response): return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_columns(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/marketdata/resources/stocks/prices.py b/src/marketdata/resources/stocks/prices.py index ad58535..6187514 100644 --- a/src/marketdata/resources/stocks/prices.py +++ b/src/marketdata/resources/stocks/prices.py @@ -11,8 +11,8 @@ from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.stocks_prices import StockPrice, StockPricesHumanReadable from marketdata.params import universal_params -from marketdata.resources.base import model_errors, no_data_result -from marketdata.utils import get_data_records, is_no_data, parse_json +from marketdata.resources.base import _parse_json_columns, model_errors, no_data_result +from marketdata.utils import get_data_records, is_no_data @api_error_handler(service="/v1/stocks/prices/") @@ -61,20 +61,22 @@ def prices( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_columns(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["symbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - fields, extra = _split_fields(output_model, parse_json(response, exact=True)) + fields, extra = _split_fields( + output_model, _parse_json_columns(response, exact=True) + ) data = get_data_records(fields) with model_errors(response): return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_columns(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/marketdata/resources/stocks/quotes.py b/src/marketdata/resources/stocks/quotes.py index e3f46af..f231b3d 100644 --- a/src/marketdata/resources/stocks/quotes.py +++ b/src/marketdata/resources/stocks/quotes.py @@ -8,8 +8,13 @@ from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.output_types.stocks_quotes import StockQuote, StockQuotesHumanReadable from marketdata.params import universal_params -from marketdata.resources.base import BaseResource, model_errors, no_data_result -from marketdata.utils import get_data_records, is_no_data, parse_json +from marketdata.resources.base import ( + BaseResource, + _parse_json_columns, + model_errors, + no_data_result, +) +from marketdata.utils import get_data_records, is_no_data @api_error_handler(service="/v1/stocks/quotes/") @@ -61,20 +66,22 @@ def quotes( ) if user_universal_params.output_format == OutputFormat.DATAFRAME: - data = parse_json(response) + data = _parse_json_columns(response) handler = get_dataframe_output_handler() return handler(data, output_model, user_universal_params).get_result( index_columns=["symbol", "Symbol"] ) elif user_universal_params.output_format == OutputFormat.INTERNAL: - fields, extra = _split_fields(output_model, parse_json(response, exact=True)) + fields, extra = _split_fields( + output_model, _parse_json_columns(response, exact=True) + ) data = get_data_records(fields, exclude_keys=["s"]) with model_errors(response): return _with_row_extra([output_model(**row) for row in data], extra) elif user_universal_params.output_format == OutputFormat.JSON: - return parse_json(response) + return _parse_json_columns(response) elif user_universal_params.output_format == OutputFormat.CSV: return user_universal_params.write_file(response.text) diff --git a/src/marketdata/resources/utilities/base.py b/src/marketdata/resources/utilities/base.py index 02f8702..e9ccb2e 100644 --- a/src/marketdata/resources/utilities/base.py +++ b/src/marketdata/resources/utilities/base.py @@ -16,7 +16,13 @@ from marketdata.output_handlers import get_dataframe_output_handler from marketdata.output_types.columns import _split_fields, _with_row_extra from marketdata.resources.base import BaseResource, model_errors, no_data_result -from marketdata.utils import dict_to_csv, get_data_records, is_no_data, parse_json +from marketdata.utils import ( + _check_column_lengths, + dict_to_csv, + get_data_records, + is_no_data, + parse_json, +) def resolve_output_params( @@ -63,7 +69,8 @@ def render( answer. The empty result for a ``no_data`` answer. Raises: - ParseError: If the body is not valid JSON or the model refuses it. + ParseError: If the body is not valid JSON, a column-oriented answer + holds lists of different lengths, or the model refuses it. """ output_format = user_universal_params.output_format @@ -76,6 +83,8 @@ def render( response=response, ) data = parse_json(response) + if as_records: + _check_column_lengths(response, data) if output_format == OutputFormat.DATAFRAME: handler = get_dataframe_output_handler() diff --git a/src/marketdata/utils.py b/src/marketdata/utils.py index 6a0b662..7db6772 100644 --- a/src/marketdata/utils.py +++ b/src/marketdata/utils.py @@ -511,12 +511,37 @@ def json_answer_columns( not_lists = [key for key in columns if not isinstance(answer[key], list)] if not_lists: raise parse_error(response, f"columns {not_lists!r} are not lists") - lengths = {key: len(answer[key]) for key in columns} - if len(set(lengths.values())) > 1: - raise parse_error(response, f"columns of different lengths {lengths!r}") + _check_column_lengths(response, answer, columns) return columns +def _check_column_lengths( + response: Response, answer: Any, columns: list[str] | None = None +) -> None: + """Refuse a column-oriented answer whose columns differ in length. + + Args: + response: The answer, named in the error. + answer: Its decoded body. A body that is not an object passes. + columns: The keys to compare, in the order the error lists them; by + default every key but the status flag ``s`` that holds a list. + + Raises: + ParseError: If two of those keys hold lists of different lengths. + """ + if not isinstance(answer, dict): + return + if columns is None: + columns = [ + key + for key, value in answer.items() + if key != "s" and isinstance(value, list) + ] + lengths = {key: len(answer[key]) for key in columns} + if len(set(lengths.values())) > 1: + raise parse_error(response, f"columns of different lengths {lengths!r}") + + _ONE_DAY = datetime.timedelta(days=1) diff --git a/src/tests/test_money_decimal.py b/src/tests/test_money_decimal.py index a6aed11..0f88ef6 100644 --- a/src/tests/test_money_decimal.py +++ b/src/tests/test_money_decimal.py @@ -821,25 +821,111 @@ def test_a_column_no_model_declares_reaches_every_format( assert "brandNew" in header.split(",") -@pytest.mark.parametrize( - "value", - ["new", ["one", "two", "three"]], - ids=["one value", "a list of another length"], -) -def test_an_undeclared_column_without_one_value_per_row_goes_to_every_row( - respx_mock, client, value -): - """A column the model does not declare that does not hold one value per - row is given to every row as the API sent it.""" +def test_an_undeclared_value_that_is_not_a_list_goes_to_every_row(respx_mock, client): + """A column the model does not declare that holds one value instead of a + list gives that value to every row, on every format that decodes it.""" case = MODEL_CASES["funds.candles"] data = _load_fixture(case.fixture) - data["brandNew"] = value + data["brandNew"] = "new" _respond(respx_mock, case, json.dumps(data).encode()) rows = case.call(client, output_format=OutputFormat.INTERNAL) + as_json = case.call(client, output_format=OutputFormat.JSON) + frame = case.call(client, output_format=OutputFormat.DATAFRAME) + + assert [marketdata.get_extra(row) for row in rows] == [{"brandNew": "new"}] * 7 + assert as_json["brandNew"] == "new" + assert list(frame["brandNew"]) == ["new"] * 7 + + +def _list_columns(data: dict) -> list[str]: + """Name the keys of a decoded answer that hold a list, but ``s``. + + Args: + data: The decoded answer. + + Returns: + The keys, in the answer's order. + """ + return [ + key for key, value in data.items() if key != "s" and isinstance(value, list) + ] + + +def _parse_errors_on_decoded_formats( + client, case: Case, path: pathlib.Path +) -> set[str]: + """Call a resource on every format that decodes its answer. + + Args: + client: The client. + case: The resource. + path: The file ``utilities.status`` writes its CSV to, from the decoded + answer. + + Returns: + The messages of the ``ParseError`` each call raised. + """ + calls = [ + {"output_format": OutputFormat.INTERNAL}, + {"output_format": OutputFormat.JSON}, + {"output_format": OutputFormat.DATAFRAME}, + ] + if case is MODEL_CASES["utilities.status"]: + calls.append({"output_format": OutputFormat.CSV, "filename": path}) + messages = set() + for kwargs in calls: + with pytest.raises(ParseError) as failure: + case.call(client, **kwargs) + messages.add(failure.value.message) + return messages + + +UNEVEN_CASES = [name for name in MODEL_CASES if name != "options.lookup"] +SHORT_CASES = [name for name in UNEVEN_CASES if "expirations" not in name] + + +@pytest.mark.parametrize("name", UNEVEN_CASES) +def test_an_undeclared_list_of_another_length_is_a_parse_error_on_every_format( + respx_mock, client, tmp_path, name +): + """A column the model does not declare whose list has one value more than + the rows raises one ``ParseError`` on INTERNAL, JSON and DataFrame, and on + the CSV ``utilities.status`` writes from the decoded answer.""" + case = MODEL_CASES[name] + data = _load_fixture(case.fixture) + rows = len(data[_list_columns(data)[0]]) + data["brandNew"] = list(range(rows + 1)) + _respond(respx_mock, case, json.dumps(data).encode()) + + messages = _parse_errors_on_decoded_formats(client, case, tmp_path / "a.csv") + + assert len(messages) == 1 + message = messages.pop() + assert "(columns of different lengths {" in message + assert f"'brandNew': {rows + 1}}})" in message + + +@pytest.mark.parametrize("name", SHORT_CASES) +def test_a_declared_column_of_another_length_is_a_parse_error_on_every_format( + respx_mock, client, tmp_path, name +): + """A column the model declares that is one value short of the others + raises one ``ParseError`` on INTERNAL, JSON and DataFrame, and on the CSV + ``utilities.status`` writes from the decoded answer.""" + case = MODEL_CASES[name] + data = _load_fixture(case.fixture) + first = _list_columns(data)[0] + rows = len(data[first]) + data[first] = data[first][:-1] + _respond(respx_mock, case, json.dumps(data).encode()) + + messages = _parse_errors_on_decoded_formats(client, case, tmp_path / "a.csv") - assert len(rows) == 7 - assert [marketdata.get_extra(row) for row in rows] == [{"brandNew": value}] * 7 + assert len(messages) == 1 + message = messages.pop() + assert "(columns of different lengths {" in message + assert f"{first!r}: {rows - 1}" in message def test_get_extra_is_empty_for_an_object_with_no_dict():