diff --git a/.github/workflows/test-and-deploy.yml b/.github/workflows/test-and-deploy.yml index 34e8b3c..b607a63 100644 --- a/.github/workflows/test-and-deploy.yml +++ b/.github/workflows/test-and-deploy.yml @@ -21,6 +21,11 @@ jobs: - run: pip install . - run: numerai copy-example - run: test -e tournament-python3/predict.py + - name: Test Signals example + if: matrix.python-version == '3.13' + run: | + pip install -r numerai/examples/signals-python3/requirements.txt + python -m unittest discover -s tests -v build-docker: name: Build Docker diff --git a/numerai/examples/signals-python3/predict.py b/numerai/examples/signals-python3/predict.py index 0a8b710..1163b72 100644 --- a/numerai/examples/signals-python3/predict.py +++ b/numerai/examples/signals-python3/predict.py @@ -1,7 +1,6 @@ """ Sample tournament model in python 3 """ import os -import json import logging import joblib import numerapi @@ -11,9 +10,10 @@ logging.basicConfig(filename="log.txt", filemode="a") TOURNAMENT = 11 -DATA_VERSION = "signals/v2.1" +DATA_VERSION = "signals/v3.0" TARGET_COL = "target" -TRAINED_MODEL_PREFIX = "./trained_model" +# Keep models trained on older Signals datasets separate from this version. +TRAINED_MODEL_PREFIX = f"./trained_model_{DATA_VERSION.replace('/', '_')}" DEFAULT_MODEL_ID = None DEFAULT_PUBLIC_ID = None @@ -59,7 +59,7 @@ def train(napi, model_id, force_training=False): num_leaves=2**5 - 1, colsample_bytree=0.1, ) - model.fit(train_data[feature_cols], train_data["target"]) + model.fit(train_data[feature_cols], train_data[TARGET_COL]) logging.info("saving model") joblib.dump(model, model_name) @@ -72,16 +72,8 @@ def predict(napi, model): predict_data = pd.read_parquet(f"{DATA_VERSION}/live.parquet").set_index( 'numerai_ticker' ) - feature_cols = [ - col - for col in predict_data.columns - if col.startswith('feature_') - and col not in ("feature_country", "feature_exchange_code") - ] - print(predict_data) - logging.info("generating predictions") - predictions = model.predict(predict_data[feature_cols]) + predictions = model.predict(predict_data[model.feature_name_]) predictions = pd.DataFrame( predictions, columns=["prediction"], index=predict_data.index ) @@ -92,7 +84,6 @@ def submit(predictions, predict_output_path="predictions.csv", model_id=None): logging.info("writing predictions to file and submitting") include_index = predictions.index.name is not None predictions.to_csv(predict_output_path, index=include_index) - print(predictions) napi.upload_predictions(predict_output_path, model_id=model_id) diff --git a/numerai/examples/signals-python3/train.py b/numerai/examples/signals-python3/train.py index 005d875..f2d0142 100644 --- a/numerai/examples/signals-python3/train.py +++ b/numerai/examples/signals-python3/train.py @@ -2,9 +2,4 @@ import predict -train_data_path, predict_data_path, predict_output_path = predict.download_data() - -model_id = predict.MODEL_ID -model_type = predict.MODEL - -predict.train(train_data_path, model_id, model_type, force_training=True) +predict.train(predict.napi, predict.MODEL_ID, force_training=True) diff --git a/tests/test_signals_example.py b/tests/test_signals_example.py new file mode 100644 index 0000000..ef2633f --- /dev/null +++ b/tests/test_signals_example.py @@ -0,0 +1,74 @@ +"""Checks for the deployable Signals example without downloading full datasets.""" + +import importlib.util +from pathlib import Path +import runpy +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +import pandas as pd + + +EXAMPLE = Path(__file__).resolve().parents[1] / "numerai/examples/signals-python3/predict.py" +spec = importlib.util.spec_from_file_location("signals_example", EXAMPLE) +signals = importlib.util.module_from_spec(spec) +with patch("logging.basicConfig"): + spec.loader.exec_module(signals) + + +class SignalsExampleTest(unittest.TestCase): + def test_training_entry_point(self): + example_dir = EXAMPLE.parent + with patch.dict(sys.modules, {"predict": signals}), patch.object(signals, "train") as train: + runpy.run_path(str(example_dir / "train.py"), run_name="__main__") + train.assert_called_once_with(signals.napi, signals.MODEL_ID, force_training=True) + + def test_train_and_predict_with_v3_data(self): + train_data = pd.DataFrame( + { + "feature_alpha": [0.1, 0.2], + "feature_beta": [0.3, 0.4], + "feature_country": ["US", "GB"], + "target": [0.2, 0.8], + } + ) + live_data = pd.DataFrame( + { + "numerai_ticker": ["A", "B"], + "feature_beta": [0.5, 0.6], + "feature_alpha": [0.7, 0.8], + "feature_country": ["US", "GB"], + } + ) + api = Mock() + model = Mock() + model.feature_name_ = ["feature_alpha", "feature_beta"] + model.predict.return_value = [0.1, 0.9] + + def read_parquet(path): + return train_data if path.endswith("train.parquet") else live_data + + with tempfile.TemporaryDirectory() as directory: + with ( + patch.object(signals, "TRAINED_MODEL_PREFIX", f"{directory}/trained_model_signals_v3.0"), + patch.object(signals.pd, "read_parquet", side_effect=read_parquet), + patch.object(signals.lgbm, "LGBMRegressor", return_value=model), + patch.object(signals.joblib, "dump"), + ): + trained_model = signals.train(api, "model-id") + predictions = signals.predict(api, trained_model) + + self.assertEqual( + [call.args[0] for call in api.download_dataset.call_args_list], + ["signals/v3.0/train.parquet", "signals/v3.0/live.parquet"], + ) + self.assertEqual(list(model.fit.call_args.args[0].columns), model.feature_name_) + self.assertEqual(list(model.predict.call_args.args[0].columns), model.feature_name_) + self.assertEqual(predictions.index.name, "numerai_ticker") + self.assertEqual(predictions["prediction"].tolist(), [0.1, 0.9]) + + +if __name__ == "__main__": + unittest.main()