From 32ad5534770f8b9836284d58d03cbb51dafe50e6 Mon Sep 17 00:00:00 2001 From: Robert Tidball Date: Wed, 8 Jul 2026 16:05:07 +1000 Subject: [PATCH 1/6] Add FXMacroData forex data source --- backend_api_python/app/config/__init__.py | 2 + backend_api_python/app/config/data_sources.py | 21 ++++++ backend_api_python/app/data_sources/forex.py | 68 +++++++++++++++++-- .../tests/test_fxmacrodata_forex_source.py | 50 ++++++++++++++ 4 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 backend_api_python/tests/test_fxmacrodata_forex_source.py diff --git a/backend_api_python/app/config/__init__.py b/backend_api_python/app/config/__init__.py index 705d12d9c..c3f042efc 100644 --- a/backend_api_python/app/config/__init__.py +++ b/backend_api_python/app/config/__init__.py @@ -7,6 +7,7 @@ FinnhubConfig, TradingEconomicsConfig, TiingoConfig, + FXMacroDataConfig, YFinanceConfig, CCXTConfig, AkshareConfig @@ -24,6 +25,7 @@ 'FinnhubConfig', 'TradingEconomicsConfig', 'TiingoConfig', + 'FXMacroDataConfig', 'YFinanceConfig', 'CCXTConfig', 'AkshareConfig', diff --git a/backend_api_python/app/config/data_sources.py b/backend_api_python/app/config/data_sources.py index b95f05d0c..29f5bcad5 100644 --- a/backend_api_python/app/config/data_sources.py +++ b/backend_api_python/app/config/data_sources.py @@ -283,6 +283,27 @@ class TiingoConfig(metaclass=MetaTiingoConfig): pass +class MetaFXMacroDataConfig(type): + @property + def BASE_URL(cls): + return _config_str('fxmacrodata', 'base_url', 'FXMACRODATA_BASE_URL', 'https://fxmacrodata.com/api/v1').rstrip('/') + + @property + def API_KEY(cls): + return _config_str('fxmacrodata', 'api_key', 'FXMACRODATA_API_KEY') or _config_str( + 'fxmacrodata', 'api_key', 'FXMD_API_KEY' + ) + + @property + def TIMEOUT(cls): + return _config_int('fxmacrodata', 'timeout', 'FXMACRODATA_TIMEOUT', 12) + + +class FXMacroDataConfig(metaclass=MetaFXMacroDataConfig): + """FXMacroData daily FX reference-rate configuration.""" + pass + + class MetaYFinanceConfig(type): @property def TIMEOUT(cls): diff --git a/backend_api_python/app/data_sources/forex.py b/backend_api_python/app/data_sources/forex.py index 59fdbaa67..e5ca5bb0b 100644 --- a/backend_api_python/app/data_sources/forex.py +++ b/backend_api_python/app/data_sources/forex.py @@ -1,9 +1,9 @@ """ 外汇数据源 -三级降级: Twelve Data → Tiingo → yfinance +四级降级: FXMacroData(日线参考汇率) → Twelve Data → Tiingo → yfinance """ from typing import Dict, List, Any, Optional -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import os import time import requests @@ -12,7 +12,7 @@ from app.data_sources.base import BaseDataSource, TIMEFRAME_SECONDS from app.utils.logger import get_logger -from app.config import TiingoConfig, APIKeys +from app.config import TiingoConfig, FXMacroDataConfig, APIKeys logger = get_logger(__name__) @@ -321,10 +321,11 @@ def get_kline( ) -> List[Dict[str, Any]]: """ 获取外汇K线数据 - Priority: Twelve Data → Tiingo → yfinance + Priority: FXMacroData (daily reference rates) → Twelve Data → Tiingo → yfinance """ symbol = normalize_forex_pair_symbol(symbol) for fetcher in ( + self._get_kline_fxmacrodata, self._get_kline_twelvedata, self._get_kline_tiingo, self._get_kline_yfinance, @@ -343,6 +344,65 @@ def get_kline( logger.debug("Forex kline fetcher %s failed for %s: %s", fetcher.__name__, symbol, e) return [] + def _get_kline_fxmacrodata( + self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None + ) -> List[Dict[str, Any]]: + """Fetch daily FX reference-rate bars from FXMacroData.""" + if timeframe != '1D': + return [] + normalized = normalize_forex_pair_symbol(symbol) + if len(normalized) != 6 or not normalized.isalpha(): + return [] + + if before_time: + end_dt = datetime.fromtimestamp(int(before_time), tz=timezone.utc) + else: + end_dt = datetime.now(tz=timezone.utc) + start_dt = end_dt - timedelta(days=max(int(limit), 1) * 2) + + params: Dict[str, Any] = { + "start_date": start_dt.strftime("%Y-%m-%d"), + "end_date": end_dt.strftime("%Y-%m-%d"), + } + if FXMacroDataConfig.API_KEY: + params["api_key"] = FXMacroDataConfig.API_KEY + + url = f"{FXMacroDataConfig.BASE_URL}/forex/{normalized[:3].lower()}/{normalized[3:].lower()}" + try: + response = requests.get(url, params=params, timeout=FXMacroDataConfig.TIMEOUT) + response.raise_for_status() + data = response.json() + except requests.exceptions.RequestException as e: + logger.debug("FXMacroData forex kline request failed %s: %s", symbol, e) + return [] + + rows = data.get("data") if isinstance(data, dict) else None + if not isinstance(rows, list): + logger.debug("FXMacroData forex kline response missing data list for %s", symbol) + return [] + + klines = [] + for row in rows: + try: + dt = datetime.fromisoformat(str(row["date"])).replace(tzinfo=timezone.utc) + price = float(row["val"]) + klines.append({ + "time": int(dt.timestamp()), + "open": price, + "high": price, + "low": price, + "close": price, + "volume": 0.0, + }) + except Exception: + continue + + klines.sort(key=lambda x: x["time"]) + if len(klines) > limit: + klines = klines[-limit:] + logger.debug("FXMacroData forex kline %s %s: %d bars", symbol, timeframe, len(klines)) + return klines + def _get_kline_twelvedata( self, symbol: str, timeframe: str, limit: int, before_time: Optional[int] = None ) -> List[Dict[str, Any]]: diff --git a/backend_api_python/tests/test_fxmacrodata_forex_source.py b/backend_api_python/tests/test_fxmacrodata_forex_source.py new file mode 100644 index 000000000..16c6d7a17 --- /dev/null +++ b/backend_api_python/tests/test_fxmacrodata_forex_source.py @@ -0,0 +1,50 @@ +from app.data_sources.forex import ForexDataSource + + +def test_fxmacrodata_daily_kline_fetch(monkeypatch): + captured = {} + + class FakeResponse: + @staticmethod + def raise_for_status(): + return None + + @staticmethod + def json(): + return { + "data": [ + {"date": "2024-01-03", "val": 1.0920}, + {"date": "2024-01-01", "val": "1.1038"}, + ] + } + + def fake_get(url, params, timeout): + captured["url"] = url + captured["params"] = params + captured["timeout"] = timeout + return FakeResponse() + + monkeypatch.setenv("FXMACRODATA_API_KEY", "test-key") + monkeypatch.setattr("app.data_sources.forex.requests.get", fake_get) + source = ForexDataSource() + + rows = source._get_kline_fxmacrodata("EUR/USD", "1D", 5, before_time=1706745600) + + assert rows == [ + {"time": 1704067200, "open": 1.1038, "high": 1.1038, "low": 1.1038, "close": 1.1038, "volume": 0.0}, + {"time": 1704240000, "open": 1.092, "high": 1.092, "low": 1.092, "close": 1.092, "volume": 0.0}, + ] + assert captured == { + "url": "https://fxmacrodata.com/api/v1/forex/eur/usd", + "params": { + "start_date": "2024-01-22", + "end_date": "2024-02-01", + "api_key": "test-key", + }, + "timeout": 12, + } + + +def test_fxmacrodata_skips_intraday_timeframes(): + source = ForexDataSource() + assert source._get_kline_fxmacrodata("EURUSD", "1m", 5) == [] From b03cda809bd7b6ac1fc85b2947dd5b7403f8aad1 Mon Sep 17 00:00:00 2001 From: Robert Tidball Date: Wed, 26 Aug 2026 12:12:52 +1000 Subject: [PATCH 2/6] Use the documented FXMacroData API host Switches the base URL from the undocumented fxmacrodata.com/api/v1 alias to the published api.fxmacrodata.com/v1 host, so the client matches the public API reference. --- backend_api_python/app/config/data_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend_api_python/app/config/data_sources.py b/backend_api_python/app/config/data_sources.py index 29f5bcad5..c70cd3a70 100644 --- a/backend_api_python/app/config/data_sources.py +++ b/backend_api_python/app/config/data_sources.py @@ -286,7 +286,7 @@ class TiingoConfig(metaclass=MetaTiingoConfig): class MetaFXMacroDataConfig(type): @property def BASE_URL(cls): - return _config_str('fxmacrodata', 'base_url', 'FXMACRODATA_BASE_URL', 'https://fxmacrodata.com/api/v1').rstrip('/') + return _config_str('fxmacrodata', 'base_url', 'FXMACRODATA_BASE_URL', 'https://api.fxmacrodata.com/v1').rstrip('/') @property def API_KEY(cls): From b67fa6e925415f8ca4a464cdb3b21fed29d97a78 Mon Sep 17 00:00:00 2001 From: Robert Tidball Date: Wed, 26 Aug 2026 12:12:53 +1000 Subject: [PATCH 3/6] Use the documented FXMacroData API host Switches the base URL from the undocumented fxmacrodata.com/api/v1 alias to the published api.fxmacrodata.com/v1 host, so the client matches the public API reference. --- backend_api_python/tests/test_fxmacrodata_forex_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend_api_python/tests/test_fxmacrodata_forex_source.py b/backend_api_python/tests/test_fxmacrodata_forex_source.py index 16c6d7a17..f38e21294 100644 --- a/backend_api_python/tests/test_fxmacrodata_forex_source.py +++ b/backend_api_python/tests/test_fxmacrodata_forex_source.py @@ -35,7 +35,7 @@ def fake_get(url, params, timeout): {"time": 1704240000, "open": 1.092, "high": 1.092, "low": 1.092, "close": 1.092, "volume": 0.0}, ] assert captured == { - "url": "https://fxmacrodata.com/api/v1/forex/eur/usd", + "url": "https://api.fxmacrodata.com/v1/forex/eur/usd", "params": { "start_date": "2024-01-22", "end_date": "2024-02-01", From 27551db5686d60b16a4b74513fd89ee425507428 Mon Sep 17 00:00:00 2001 From: Robert Tidball Date: Mon, 28 Sep 2026 12:57:26 +1000 Subject: [PATCH 4/6] Send the FXMacroData key in the X-API-Key header --- backend_api_python/app/data_sources/forex.py | 5 +++-- backend_api_python/tests/test_fxmacrodata_forex_source.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend_api_python/app/data_sources/forex.py b/backend_api_python/app/data_sources/forex.py index e5ca5bb0b..6eb714902 100644 --- a/backend_api_python/app/data_sources/forex.py +++ b/backend_api_python/app/data_sources/forex.py @@ -364,12 +364,13 @@ def _get_kline_fxmacrodata( "start_date": start_dt.strftime("%Y-%m-%d"), "end_date": end_dt.strftime("%Y-%m-%d"), } + headers: Dict[str, str] = {} if FXMacroDataConfig.API_KEY: - params["api_key"] = FXMacroDataConfig.API_KEY + headers["X-API-Key"] = FXMacroDataConfig.API_KEY url = f"{FXMacroDataConfig.BASE_URL}/forex/{normalized[:3].lower()}/{normalized[3:].lower()}" try: - response = requests.get(url, params=params, timeout=FXMacroDataConfig.TIMEOUT) + response = requests.get(url, params=params, headers=headers, timeout=FXMacroDataConfig.TIMEOUT) response.raise_for_status() data = response.json() except requests.exceptions.RequestException as e: diff --git a/backend_api_python/tests/test_fxmacrodata_forex_source.py b/backend_api_python/tests/test_fxmacrodata_forex_source.py index f38e21294..3e64bd3af 100644 --- a/backend_api_python/tests/test_fxmacrodata_forex_source.py +++ b/backend_api_python/tests/test_fxmacrodata_forex_source.py @@ -18,9 +18,10 @@ def json(): ] } - def fake_get(url, params, timeout): + def fake_get(url, params, headers, timeout): captured["url"] = url captured["params"] = params + captured["headers"] = headers captured["timeout"] = timeout return FakeResponse() @@ -39,8 +40,8 @@ def fake_get(url, params, timeout): "params": { "start_date": "2024-01-22", "end_date": "2024-02-01", - "api_key": "test-key", }, + "headers": {"X-API-Key": "test-key"}, "timeout": 12, } From 359b028d811438095521262dca2ef0277af6d513 Mon Sep 17 00:00:00 2001 From: Robert Tidball Date: Mon, 28 Sep 2026 13:46:57 +1000 Subject: [PATCH 5/6] Page through FXMacroData forex history for daily klines --- backend_api_python/app/data_sources/forex.py | 38 +++++++++++++------ .../tests/test_fxmacrodata_forex_source.py | 36 ++++++++++++++++++ 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/backend_api_python/app/data_sources/forex.py b/backend_api_python/app/data_sources/forex.py index 6eb714902..e47ba6bfb 100644 --- a/backend_api_python/app/data_sources/forex.py +++ b/backend_api_python/app/data_sources/forex.py @@ -369,18 +369,34 @@ def _get_kline_fxmacrodata( headers["X-API-Key"] = FXMacroDataConfig.API_KEY url = f"{FXMacroDataConfig.BASE_URL}/forex/{normalized[:3].lower()}/{normalized[3:].lower()}" - try: - response = requests.get(url, params=params, headers=headers, timeout=FXMacroDataConfig.TIMEOUT) - response.raise_for_status() - data = response.json() - except requests.exceptions.RequestException as e: - logger.debug("FXMacroData forex kline request failed %s: %s", symbol, e) - return [] + # The API returns at most 100 rows per request, newest first; page + # with offset until enough bars are collected or has_more is false. + rows: List[Dict[str, Any]] = [] + offset = 0 + for _ in range(100): + page_params = dict(params, limit=100, offset=offset) + try: + response = requests.get(url, params=page_params, headers=headers, timeout=FXMacroDataConfig.TIMEOUT) + response.raise_for_status() + data = response.json() + except requests.exceptions.RequestException as e: + logger.debug("FXMacroData forex kline request failed %s: %s", symbol, e) + return [] - rows = data.get("data") if isinstance(data, dict) else None - if not isinstance(rows, list): - logger.debug("FXMacroData forex kline response missing data list for %s", symbol) - return [] + page = data.get("data") if isinstance(data, dict) else None + if not isinstance(page, list): + logger.debug("FXMacroData forex kline response missing data list for %s", symbol) + return [] + rows.extend(page) + pagination = data.get("pagination") or {} + if not page or len(rows) >= limit or not pagination.get("has_more"): + break + next_offset = pagination.get("next_offset") + if next_offset is None: + next_offset = offset + len(page) + if next_offset <= offset: + break + offset = next_offset klines = [] for row in rows: diff --git a/backend_api_python/tests/test_fxmacrodata_forex_source.py b/backend_api_python/tests/test_fxmacrodata_forex_source.py index 3e64bd3af..df3151d1c 100644 --- a/backend_api_python/tests/test_fxmacrodata_forex_source.py +++ b/backend_api_python/tests/test_fxmacrodata_forex_source.py @@ -40,12 +40,48 @@ def fake_get(url, params, headers, timeout): "params": { "start_date": "2024-01-22", "end_date": "2024-02-01", + "limit": 100, + "offset": 0, }, "headers": {"X-API-Key": "test-key"}, "timeout": 12, } +def test_fxmacrodata_daily_kline_follows_pagination(monkeypatch): + pages = { + 0: { + "data": [{"date": "2024-01-03", "val": 1.3}, {"date": "2024-01-02", "val": 1.2}], + "pagination": {"has_more": True, "next_offset": 2}, + }, + 2: { + "data": [{"date": "2024-01-01", "val": 1.1}], + "pagination": {"has_more": False, "next_offset": None}, + }, + } + offsets = [] + + class FakeResponse: + def __init__(self, payload): + self.payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self.payload + + def fake_get(url, params, headers, timeout): + offsets.append(params["offset"]) + return FakeResponse(pages[params["offset"]]) + + monkeypatch.setattr("app.data_sources.forex.requests.get", fake_get) + rows = ForexDataSource()._get_kline_fxmacrodata("EURUSD", "1D", 5, before_time=1704326400) + + assert offsets == [0, 2] + assert [row["close"] for row in rows] == [1.1, 1.2, 1.3] + + def test_fxmacrodata_skips_intraday_timeframes(): source = ForexDataSource() assert source._get_kline_fxmacrodata("EURUSD", "1m", 5) == [] From 43226fa7578c1058c1984e349f9122ac6e5ad27d Mon Sep 17 00:00:00 2001 From: Robert Tidball Date: Mon, 5 Oct 2026 20:58:14 +1100 Subject: [PATCH 6/6] Do not follow redirects on FXMacroData forex requests --- backend_api_python/app/data_sources/forex.py | 13 +++++++- .../tests/test_fxmacrodata_forex_source.py | 33 +++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/backend_api_python/app/data_sources/forex.py b/backend_api_python/app/data_sources/forex.py index e47ba6bfb..a22b3e5e0 100644 --- a/backend_api_python/app/data_sources/forex.py +++ b/backend_api_python/app/data_sources/forex.py @@ -376,7 +376,18 @@ def _get_kline_fxmacrodata( for _ in range(100): page_params = dict(params, limit=100, offset=offset) try: - response = requests.get(url, params=page_params, headers=headers, timeout=FXMacroDataConfig.TIMEOUT) + # Do not follow redirects, so the X-API-Key header is never + # forwarded to another host. + response = requests.get( + url, + params=page_params, + headers=headers, + timeout=FXMacroDataConfig.TIMEOUT, + allow_redirects=False, + ) + if 300 <= response.status_code < 400: + logger.debug("FXMacroData forex kline request redirected for %s; not followed", symbol) + return [] response.raise_for_status() data = response.json() except requests.exceptions.RequestException as e: diff --git a/backend_api_python/tests/test_fxmacrodata_forex_source.py b/backend_api_python/tests/test_fxmacrodata_forex_source.py index df3151d1c..ff1447469 100644 --- a/backend_api_python/tests/test_fxmacrodata_forex_source.py +++ b/backend_api_python/tests/test_fxmacrodata_forex_source.py @@ -5,6 +5,8 @@ def test_fxmacrodata_daily_kline_fetch(monkeypatch): captured = {} class FakeResponse: + status_code = 200 + @staticmethod def raise_for_status(): return None @@ -18,7 +20,7 @@ def json(): ] } - def fake_get(url, params, headers, timeout): + def fake_get(url, params, headers, timeout, **kwargs): captured["url"] = url captured["params"] = params captured["headers"] = headers @@ -62,6 +64,8 @@ def test_fxmacrodata_daily_kline_follows_pagination(monkeypatch): offsets = [] class FakeResponse: + status_code = 200 + def __init__(self, payload): self.payload = payload @@ -71,7 +75,7 @@ def raise_for_status(self): def json(self): return self.payload - def fake_get(url, params, headers, timeout): + def fake_get(url, params, headers, timeout, **kwargs): offsets.append(params["offset"]) return FakeResponse(pages[params["offset"]]) @@ -85,3 +89,28 @@ def fake_get(url, params, headers, timeout): def test_fxmacrodata_skips_intraday_timeframes(): source = ForexDataSource() assert source._get_kline_fxmacrodata("EURUSD", "1m", 5) == [] + + +def test_fxmacrodata_does_not_follow_redirects(monkeypatch): + captured = {} + + class FakeResponse: + status_code = 302 + + @staticmethod + def raise_for_status(): + return None + + @staticmethod + def json(): + return {"data": [{"date": "2024-01-03", "val": 1.0920}]} + + def fake_get(url, params, headers, timeout, **kwargs): + captured.update(kwargs) + return FakeResponse() + + monkeypatch.setenv("FXMACRODATA_API_KEY", "test-key") + monkeypatch.setattr("app.data_sources.forex.requests.get", fake_get) + + assert ForexDataSource()._get_kline_fxmacrodata("EURUSD", "1D", 5) == [] + assert captured["allow_redirects"] is False