From ed795b07b0347fe79d8e8c5eee1cb272aaa2d055 Mon Sep 17 00:00:00 2001 From: Tony Bagnall Date: Thu, 3 Sep 2026 19:15:44 +0100 Subject: [PATCH] Reserve the DisjointCNN name for the port, move aeon's results aside aeon's results are renamed to DisjointCNN-Aeon and kept, rather than deleted: they are the evidence for aeon issue #3775, where a Permute after the final block leaves GlobalAveragePooling2D reducing the wrong axes so the classifier head receives one feature instead of 64, putting it about 20 accuracy points below the published numbers on all 23 shared datasets. They stay excluded from the leaderboard for the same reason as before, that listing them would read as a claim about the method rather than about one implementation. The exclusion and its explanation move with the name. DisjointCNN is now free for the port, which follows the authors' training procedure and reaches 0.952 on ERing against 0.964 published, where aeon manages 0.641. Nothing reports under that name until the run lands, so the leaderboard has no DisjointCNN row in the meantime. Co-Authored-By: Claude Opus 5 --- multiverse/experiments/tables.py | 14 ++++++++------ .../DisjointCNN-Aeon_accuracy.csv} | 0 .../DisjointCNN-Aeon_auroc.csv} | 0 .../DisjointCNN-Aeon_balacc.csv} | 0 .../DisjointCNN-Aeon_f1.csv} | 0 .../DisjointCNN-Aeon_logloss.csv} | 0 .../DisjointCNN-Aeon_sensitivity.csv} | 0 .../DisjointCNN-Aeon_specificity.csv} | 0 results/multiverse/missing_results.csv | 2 +- 9 files changed, 9 insertions(+), 7 deletions(-) rename results/multiverse/{DisjointCNN/DisjointCNN_accuracy.csv => DisjointCNN-Aeon/DisjointCNN-Aeon_accuracy.csv} (100%) rename results/multiverse/{DisjointCNN/DisjointCNN_auroc.csv => DisjointCNN-Aeon/DisjointCNN-Aeon_auroc.csv} (100%) rename results/multiverse/{DisjointCNN/DisjointCNN_balacc.csv => DisjointCNN-Aeon/DisjointCNN-Aeon_balacc.csv} (100%) rename results/multiverse/{DisjointCNN/DisjointCNN_f1.csv => DisjointCNN-Aeon/DisjointCNN-Aeon_f1.csv} (100%) rename results/multiverse/{DisjointCNN/DisjointCNN_logloss.csv => DisjointCNN-Aeon/DisjointCNN-Aeon_logloss.csv} (100%) rename results/multiverse/{DisjointCNN/DisjointCNN_sensitivity.csv => DisjointCNN-Aeon/DisjointCNN-Aeon_sensitivity.csv} (100%) rename results/multiverse/{DisjointCNN/DisjointCNN_specificity.csv => DisjointCNN-Aeon/DisjointCNN-Aeon_specificity.csv} (100%) diff --git a/multiverse/experiments/tables.py b/multiverse/experiments/tables.py index ec14367..5333722 100644 --- a/multiverse/experiments/tables.py +++ b/multiverse/experiments/tables.py @@ -1148,16 +1148,18 @@ def main() -> None: Uses every estimator with results in the repository, including the Dummy baseline, over the Multiverse-core datasets all of them have results for. - DisjointCNN is held back. Its results are in the repository but they are - around 20 accuracy points below the authors' published numbers on all 23 - shared datasets, so the run measures aeon's implementation rather than the - method, and listing it would read as a claim about the method. Remove it - from ``exclude`` once that is resolved. + DisjointCNN-Aeon is held back. Those results are around 20 accuracy points + below the authors' published numbers on all 23 shared datasets, because + aeon's network applies a Permute after the final block and its pooling then + reduces the wrong axes, leaving the classifier head one feature instead of + 64 (aeon issue #3775). They are kept as evidence for that issue rather than + deleted, but listing them would read as a claim about the method. The + Multiverse port of the same method reports under DisjointCNN. """ from aeon.datasets.tsc_datasets import multiverse_core datasets = sorted(multiverse_core) - estimators = available_estimators(exclude=("DisjointCNN",)) + estimators = available_estimators(exclude=("DisjointCNN-Aeon",)) print(f"estimators: {', '.join(estimators)}") path = leaderboard( diff --git a/results/multiverse/DisjointCNN/DisjointCNN_accuracy.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_accuracy.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_accuracy.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_accuracy.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_auroc.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_auroc.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_auroc.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_auroc.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_balacc.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_balacc.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_balacc.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_balacc.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_f1.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_f1.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_f1.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_f1.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_logloss.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_logloss.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_logloss.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_logloss.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_sensitivity.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_sensitivity.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_sensitivity.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_sensitivity.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_specificity.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_specificity.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_specificity.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_specificity.csv diff --git a/results/multiverse/missing_results.csv b/results/multiverse/missing_results.csv index 580f9f3..749ef43 100644 --- a/results/multiverse/missing_results.csv +++ b/results/multiverse/missing_results.csv @@ -32,7 +32,7 @@ TDE,STEW,cancelled before completion,,controller report TDE,Tiselac,Time limit,,controller report TDE,USCActivity,Time limit,,controller report TSF,AustraliaRainfall_disc,not recorded,, -DisjointCNN,EmoPain,ValueError: input collection has too little variation (std <= 1e-07),"Raised by aeon input validation before fit, so it affects any aeon classifier; 2 attempts",DeepLearning/output/DisjointCNN/EmoPain/1476290-1.err +DisjointCNN-Aeon,EmoPain,ValueError: input collection has too little variation (std <= 1e-07),"Raised by aeon input validation before fit, so it affects any aeon classifier; 2 attempts",DeepLearning/output/DisjointCNN/EmoPain/1476290-1.err TimesURL,EmoPain,ValueError: input collection has too little variation (std <= 1e-07),"Inferred, not read from a log: no TimesURL job logs were copied across. aeon raises this before fit for EmoPain, so it stops every aeon classifier, and it is the recorded cause for ConvTran, PatchMTSC and DisjointCNN", TS2Vec,EmoPain,"ValueError: input collection has too little variation (std <= 1e-07)","Raised by aeon input validation before fit, so it affects any aeon classifier; 1733 case/channel pairs flagged",DeepLearning/output/TS2Vec/EmoPain TS2Vec,AustraliaRainfall_disc,Timed out at 60 hours,"Elapsed 2-12:00:00 against a 2-12:00:00 limit, using about 3 GB, so time rather than memory. The port omitted the authors' MAX_SAMPLES cap on the SVM probe, making the grid search unaffordable on large collections; fixed, and these are rerunnable",DeepLearning/output/TS2Vec/AustraliaRainfall_disc