From 6863e3a4e01295e40fb41fa89ea05d91bd4fa89c Mon Sep 17 00:00:00 2001 From: Sasha Mitchell Date: Thu, 1 Oct 2026 02:48:59 +0700 Subject: [PATCH 1/3] Keep the Loogle key count when a prediction does not parse. A four-key reference plus an unparsed prediction scored 4/5. The miss now keeps the reference key count, so that pair is 4/8. Signed-off-by: Sasha Mitchell --- .../benchmarks/loogle/calculate_metrics.py | 26 +++++++++++++------ tests/test_loogle_exact_match.py | 17 ++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 tests/test_loogle_exact_match.py diff --git a/evaluation/benchmarks/loogle/calculate_metrics.py b/evaluation/benchmarks/loogle/calculate_metrics.py index 85e2e024..78f75eb7 100644 --- a/evaluation/benchmarks/loogle/calculate_metrics.py +++ b/evaluation/benchmarks/loogle/calculate_metrics.py @@ -4,16 +4,13 @@ import ast -import nltk import pandas as pd -from bert_score import score -from nltk.translate.bleu_score import sentence_bleu -from nltk.translate.meteor_score import single_meteor_score -from rouge import Rouge # Code below is adapted from https://github.com/bigai-nlco/LooGLE/blob/main/Evaluation/automatic_metrics.py def get_bleu_score(reference, hypothesis): + from nltk.translate.bleu_score import sentence_bleu + reference, hypothesis = ( reference.replace("\n", " ").split(), hypothesis.replace("\n", " ").split(), @@ -25,12 +22,16 @@ def get_bleu_score(reference, hypothesis): def get_rouge_score(reference, hypothesis, metric="r"): + from rouge import Rouge + rouge = Rouge() rouge_ = rouge.get_scores(hyps=[hypothesis], refs=[reference])[0] return dict((key, rouge_[key][metric]) for key in ["rouge-1", "rouge-2", "rouge-l"]) def get_meteor_score(reference, hypothesis): + from nltk.translate.meteor_score import single_meteor_score + reference, hypothesis = ( reference.replace("\n", " ").split(), hypothesis.replace("\n", " ").split(), @@ -43,13 +44,19 @@ def get_exact_match(reference, hypothesis): try: reference = ast.literal_eval(reference) count = len(reference) - hypothesis = ast.literal_eval(hypothesis) - assert isinstance(hypothesis, dict) except Exception: return 0, 1 + try: + hypothesis = ast.literal_eval(hypothesis) + if not isinstance(hypothesis, dict): + return 0, count or 1 + items = reference.items() + except Exception: + return 0, count or 1 + exact_score_count = 0 - for key, value in reference.items(): + for key, value in items: if hypothesis.get(key) == value: exact_score_count += 1 return exact_score_count, count @@ -85,6 +92,9 @@ def wrapped_metric(answer, predicted_answer): def calculate_metrics(df: pd.DataFrame) -> dict: + import nltk + from bert_score import score + nltk.download("wordnet") scores: dict = {} diff --git a/tests/test_loogle_exact_match.py b/tests/test_loogle_exact_match.py new file mode 100644 index 00000000..993701f2 --- /dev/null +++ b/tests/test_loogle_exact_match.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from evaluation.benchmarks.loogle.calculate_metrics import get_exact_match + + +def test_unparsed_prediction_keeps_the_reference_key_count(): + reference = "{'a': 1, 'b': 2, 'c': 3, 'd': 4}" + + assert get_exact_match(reference, "nope") == (0, 4) + assert get_exact_match(reference, "[]") == (0, 4) + assert get_exact_match(reference, reference) == (4, 4) + assert get_exact_match("nope", reference) == (0, 1) + + matched = get_exact_match(reference, reference) + missed = get_exact_match(reference, "nope") + assert (matched[0] + missed[0]) / (matched[1] + missed[1]) == 0.5 From 99f7866c50fc738c86ef9d37e8a807ee3c24efab Mon Sep 17 00:00:00 2001 From: Sasha Mitchell Date: Thu, 1 Oct 2026 21:39:06 +0700 Subject: [PATCH 2/3] Keep only the get_exact_match change Imports stay at module scope, and the extra test file is gone. Signed-off-by: Sasha Mitchell --- tests/test_loogle_exact_match.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 tests/test_loogle_exact_match.py diff --git a/tests/test_loogle_exact_match.py b/tests/test_loogle_exact_match.py deleted file mode 100644 index 993701f2..00000000 --- a/tests/test_loogle_exact_match.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from evaluation.benchmarks.loogle.calculate_metrics import get_exact_match - - -def test_unparsed_prediction_keeps_the_reference_key_count(): - reference = "{'a': 1, 'b': 2, 'c': 3, 'd': 4}" - - assert get_exact_match(reference, "nope") == (0, 4) - assert get_exact_match(reference, "[]") == (0, 4) - assert get_exact_match(reference, reference) == (4, 4) - assert get_exact_match("nope", reference) == (0, 1) - - matched = get_exact_match(reference, reference) - missed = get_exact_match(reference, "nope") - assert (matched[0] + missed[0]) / (matched[1] + missed[1]) == 0.5 From e398037419e1d1744a5a1f11c066cd70f0558e17 Mon Sep 17 00:00:00 2001 From: Sasha Mitchell Date: Thu, 1 Oct 2026 21:39:24 +0700 Subject: [PATCH 3/3] Restore the metric imports to module scope get_exact_match is the only behavior change. Signed-off-by: Sasha Mitchell --- evaluation/benchmarks/loogle/calculate_metrics.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/evaluation/benchmarks/loogle/calculate_metrics.py b/evaluation/benchmarks/loogle/calculate_metrics.py index 78f75eb7..837330a3 100644 --- a/evaluation/benchmarks/loogle/calculate_metrics.py +++ b/evaluation/benchmarks/loogle/calculate_metrics.py @@ -4,13 +4,16 @@ import ast +import nltk import pandas as pd +from bert_score import score +from nltk.translate.bleu_score import sentence_bleu +from nltk.translate.meteor_score import single_meteor_score +from rouge import Rouge # Code below is adapted from https://github.com/bigai-nlco/LooGLE/blob/main/Evaluation/automatic_metrics.py def get_bleu_score(reference, hypothesis): - from nltk.translate.bleu_score import sentence_bleu - reference, hypothesis = ( reference.replace("\n", " ").split(), hypothesis.replace("\n", " ").split(), @@ -22,16 +25,12 @@ def get_bleu_score(reference, hypothesis): def get_rouge_score(reference, hypothesis, metric="r"): - from rouge import Rouge - rouge = Rouge() rouge_ = rouge.get_scores(hyps=[hypothesis], refs=[reference])[0] return dict((key, rouge_[key][metric]) for key in ["rouge-1", "rouge-2", "rouge-l"]) def get_meteor_score(reference, hypothesis): - from nltk.translate.meteor_score import single_meteor_score - reference, hypothesis = ( reference.replace("\n", " ").split(), hypothesis.replace("\n", " ").split(), @@ -92,9 +91,6 @@ def wrapped_metric(answer, predicted_answer): def calculate_metrics(df: pd.DataFrame) -> dict: - import nltk - from bert_score import score - nltk.download("wordnet") scores: dict = {}