diff --git a/multiverse/experiments/tables.py b/multiverse/experiments/tables.py index ec14367..5333722 100644 --- a/multiverse/experiments/tables.py +++ b/multiverse/experiments/tables.py @@ -1148,16 +1148,18 @@ def main() -> None: Uses every estimator with results in the repository, including the Dummy baseline, over the Multiverse-core datasets all of them have results for. - DisjointCNN is held back. Its results are in the repository but they are - around 20 accuracy points below the authors' published numbers on all 23 - shared datasets, so the run measures aeon's implementation rather than the - method, and listing it would read as a claim about the method. Remove it - from ``exclude`` once that is resolved. + DisjointCNN-Aeon is held back. Those results are around 20 accuracy points + below the authors' published numbers on all 23 shared datasets, because + aeon's network applies a Permute after the final block and its pooling then + reduces the wrong axes, leaving the classifier head one feature instead of + 64 (aeon issue #3775). They are kept as evidence for that issue rather than + deleted, but listing them would read as a claim about the method. The + Multiverse port of the same method reports under DisjointCNN. """ from aeon.datasets.tsc_datasets import multiverse_core datasets = sorted(multiverse_core) - estimators = available_estimators(exclude=("DisjointCNN",)) + estimators = available_estimators(exclude=("DisjointCNN-Aeon",)) print(f"estimators: {', '.join(estimators)}") path = leaderboard( diff --git a/results/multiverse/DisjointCNN/DisjointCNN_accuracy.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_accuracy.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_accuracy.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_accuracy.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_auroc.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_auroc.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_auroc.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_auroc.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_balacc.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_balacc.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_balacc.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_balacc.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_f1.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_f1.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_f1.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_f1.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_logloss.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_logloss.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_logloss.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_logloss.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_sensitivity.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_sensitivity.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_sensitivity.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_sensitivity.csv diff --git a/results/multiverse/DisjointCNN/DisjointCNN_specificity.csv b/results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_specificity.csv similarity index 100% rename from results/multiverse/DisjointCNN/DisjointCNN_specificity.csv rename to results/multiverse/DisjointCNN-Aeon/DisjointCNN-Aeon_specificity.csv diff --git a/results/multiverse/missing_results.csv b/results/multiverse/missing_results.csv index 580f9f3..749ef43 100644 --- a/results/multiverse/missing_results.csv +++ b/results/multiverse/missing_results.csv @@ -32,7 +32,7 @@ TDE,STEW,cancelled before completion,,controller report TDE,Tiselac,Time limit,,controller report TDE,USCActivity,Time limit,,controller report TSF,AustraliaRainfall_disc,not recorded,, -DisjointCNN,EmoPain,ValueError: input collection has too little variation (std <= 1e-07),"Raised by aeon input validation before fit, so it affects any aeon classifier; 2 attempts",DeepLearning/output/DisjointCNN/EmoPain/1476290-1.err +DisjointCNN-Aeon,EmoPain,ValueError: input collection has too little variation (std <= 1e-07),"Raised by aeon input validation before fit, so it affects any aeon classifier; 2 attempts",DeepLearning/output/DisjointCNN/EmoPain/1476290-1.err TimesURL,EmoPain,ValueError: input collection has too little variation (std <= 1e-07),"Inferred, not read from a log: no TimesURL job logs were copied across. aeon raises this before fit for EmoPain, so it stops every aeon classifier, and it is the recorded cause for ConvTran, PatchMTSC and DisjointCNN", TS2Vec,EmoPain,"ValueError: input collection has too little variation (std <= 1e-07)","Raised by aeon input validation before fit, so it affects any aeon classifier; 1733 case/channel pairs flagged",DeepLearning/output/TS2Vec/EmoPain TS2Vec,AustraliaRainfall_disc,Timed out at 60 hours,"Elapsed 2-12:00:00 against a 2-12:00:00 limit, using about 3 GB, so time rather than memory. The port omitted the authors' MAX_SAMPLES cap on the SVM probe, making the grid search unaffordable on large collections; fixed, and these are rerunnable",DeepLearning/output/TS2Vec/AustraliaRainfall_disc