Update Signals example for v3 data

.github/workflows/test-and-deploy.yml

@@ -21,6 +21,11 @@ jobs:
       - run: pip install .
       - run: numerai copy-example
       - run: test -e tournament-python3/predict.py
+      - name: Test Signals example
+        if: matrix.python-version == '3.13'
+        run: |
+          pip install -r numerai/examples/signals-python3/requirements.txt
+          python -m unittest discover -s tests -v
 
   build-docker:
     name: Build Docker

numerai/examples/signals-python3/predict.py

@@ -1,7 +1,6 @@
 """ Sample tournament model in python 3 """
 
 import os
-import json
 import logging
 import joblib
 import numerapi
@@ -11,9 +10,10 @@
 logging.basicConfig(filename="log.txt", filemode="a")
 
 TOURNAMENT = 11
-DATA_VERSION = "signals/v2.1"
+DATA_VERSION = "signals/v3.0"
 TARGET_COL = "target"
-TRAINED_MODEL_PREFIX = "./trained_model"
+# Keep models trained on older Signals datasets separate from this version.
+TRAINED_MODEL_PREFIX = f"./trained_model_{DATA_VERSION.replace('/', '_')}"
 
 DEFAULT_MODEL_ID = None
 DEFAULT_PUBLIC_ID = None
@@ -59,7 +59,7 @@ def train(napi, model_id, force_training=False):
         num_leaves=2**5 - 1,
         colsample_bytree=0.1,
     )
-    model.fit(train_data[feature_cols], train_data["target"])
+    model.fit(train_data[feature_cols], train_data[TARGET_COL])
 
     logging.info("saving model")
     joblib.dump(model, model_name)
@@ -72,16 +72,8 @@ def predict(napi, model):
     predict_data = pd.read_parquet(f"{DATA_VERSION}/live.parquet").set_index(
         'numerai_ticker'
     )
-    feature_cols = [
-        col
-        for col in predict_data.columns
-        if col.startswith('feature_')
-        and col not in ("feature_country", "feature_exchange_code")
-    ]
-    print(predict_data)
-
     logging.info("generating predictions")
-    predictions = model.predict(predict_data[feature_cols])
+    predictions = model.predict(predict_data[model.feature_name_])
     predictions = pd.DataFrame(
         predictions, columns=["prediction"], index=predict_data.index
     )
@@ -92,7 +84,6 @@ def submit(predictions, predict_output_path="predictions.csv", model_id=None):
     logging.info("writing predictions to file and submitting")
     include_index = predictions.index.name is not None
     predictions.to_csv(predict_output_path, index=include_index)
-    print(predictions)
     napi.upload_predictions(predict_output_path, model_id=model_id)
 
 

numerai/examples/signals-python3/train.py

@@ -2,9 +2,4 @@
 
 import predict
 
-train_data_path, predict_data_path, predict_output_path = predict.download_data()
-
-model_id = predict.MODEL_ID
-model_type = predict.MODEL
-
-predict.train(train_data_path, model_id, model_type, force_training=True)
+predict.train(predict.napi, predict.MODEL_ID, force_training=True)

tests/test_signals_example.py

@@ -0,0 +1,74 @@
+"""Checks for the deployable Signals example without downloading full datasets."""
+
+import importlib.util
+from pathlib import Path
+import runpy
+import sys
+import tempfile
+import unittest
+from unittest.mock import Mock, patch
+
+import pandas as pd
+
+
+EXAMPLE = Path(__file__).resolve().parents[1] / "numerai/examples/signals-python3/predict.py"
+spec = importlib.util.spec_from_file_location("signals_example", EXAMPLE)
+signals = importlib.util.module_from_spec(spec)
+with patch("logging.basicConfig"):
+    spec.loader.exec_module(signals)
+
+
+class SignalsExampleTest(unittest.TestCase):
+    def test_training_entry_point(self):
+        example_dir = EXAMPLE.parent
+        with patch.dict(sys.modules, {"predict": signals}), patch.object(signals, "train") as train:
+            runpy.run_path(str(example_dir / "train.py"), run_name="__main__")
+        train.assert_called_once_with(signals.napi, signals.MODEL_ID, force_training=True)
+
+    def test_train_and_predict_with_v3_data(self):
+        train_data = pd.DataFrame(
+            {
+                "feature_alpha": [0.1, 0.2],
+                "feature_beta": [0.3, 0.4],
+                "feature_country": ["US", "GB"],
+                "target": [0.2, 0.8],
+            }
+        )
+        live_data = pd.DataFrame(
+            {
+                "numerai_ticker": ["A", "B"],
+                "feature_beta": [0.5, 0.6],
+                "feature_alpha": [0.7, 0.8],
+                "feature_country": ["US", "GB"],
+            }
+        )
+        api = Mock()
+        model = Mock()
+        model.feature_name_ = ["feature_alpha", "feature_beta"]
+        model.predict.return_value = [0.1, 0.9]
+
+        def read_parquet(path):
+            return train_data if path.endswith("train.parquet") else live_data
+
+        with tempfile.TemporaryDirectory() as directory:
+            with (
+                patch.object(signals, "TRAINED_MODEL_PREFIX", f"{directory}/trained_model_signals_v3.0"),
+                patch.object(signals.pd, "read_parquet", side_effect=read_parquet),
+                patch.object(signals.lgbm, "LGBMRegressor", return_value=model),
+                patch.object(signals.joblib, "dump"),
+            ):
+                trained_model = signals.train(api, "model-id")
+                predictions = signals.predict(api, trained_model)
+
+        self.assertEqual(
+            [call.args[0] for call in api.download_dataset.call_args_list],
+            ["signals/v3.0/train.parquet", "signals/v3.0/live.parquet"],
+        )
+        self.assertEqual(list(model.fit.call_args.args[0].columns), model.feature_name_)
+        self.assertEqual(list(model.predict.call_args.args[0].columns), model.feature_name_)
+        self.assertEqual(predictions.index.name, "numerai_ticker")
+        self.assertEqual(predictions["prediction"].tolist(), [0.1, 0.9])
+
+
+if __name__ == "__main__":
+    unittest.main()