diff --git a/docs/decisions/0166-a-changed-line-of-python-no-test-notices-is-reported-for-a-month-before-it-can-fail.md b/docs/decisions/0166-a-changed-line-of-python-no-test-notices-is-reported-for-a-month-before-it-can-fail.md index bf1e6880..c1af1f02 100644 --- a/docs/decisions/0166-a-changed-line-of-python-no-test-notices-is-reported-for-a-month-before-it-can-fail.md +++ b/docs/decisions/0166-a-changed-line-of-python-no-test-notices-is-reported-for-a-month-before-it-can-fail.md @@ -51,3 +51,7 @@ A changed line that no test notices is behaviour nothing holds. The mutant run h | In the month it reads no merge group, and the job is green whatever fails | `tests/test_untested.py::test_on_a_merge_group_it_does_not_run_and_it_is_green_whatever_fails`, `tools/untested.py`; mutant: "a merge group is read in the month"; mutant: "what fails the check turns the job red" | | An uncovered survivor is warned of on its lines and told its trailer, and the counts close the log | `tests/test_untested.py::test_a_survivor_is_warned_of_on_its_lines_and_told_its_trailer`, `tools/untested.py`; mutant: "a survivor is not warned of" | | The job runs the base's tool, and no job needs it | `tests/test_untested.py::test_the_job_runs_the_bases_tool_on_both_events_and_no_job_needs_it`, `.github/workflows/untested.yml`, `.github/workflows/verify.yml`; mutant: "the advisory job checks out the head"; mutant: "verify needs the advisory job" | +| A call in the mutant lists is data only of a helper the module defines before the list, or of `replace` on a text: a builtin's call, another module's, anything read from a name an import binds, wherever it stands, a name bound to such a value, one bound twice, or changed by a walrus, a `global`, a method, a store into it, a `del` or a call of `globals` or `eval`, one no statement binds before the list, a dunder attribute, and `replace` on what a helper gives that does not say it gives a text may read the world, and are code | `tests/test_untested.py::test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers`, `tools/untested.py`; mutant: "a call of a builtin or another module's function in the mutant lists is data"; mutant: "a call of any method in the mutant lists is data"; mutant: "a replace on anything is data in the mutant lists"; mutant: "a name no statement binds before the mutant lists is data"; mutant: "a name an import in another statement binds is data"; mutant: "a dotted import binds its whole path"; mutant: "a name another unpacking binds is data"; mutant: "a value read from code is data"; mutant: "a walrus leaves a name as it was in the mutant lists"; mutant: "a global in a helper leaves a name as it was in the mutant lists"; mutant: "a call of globals or eval leaves every name as it was in the mutant lists"; mutant: "every name is read as it was after a call of globals or eval in the mutant lists"; mutant: "a method called on a name leaves it as it was in the mutant lists"; mutant: "a method a function calls on its own name changes the module's in the mutant lists"; mutant: "a decorated helper is a helper in the mutant lists"; mutant: "a dunder attribute is data in the mutant lists"; mutant: "any helper gives a text in the mutant lists"; mutant: "an annotated assignment is code in the mutant lists"; mutant: "a call whose result is read as a text need not itself be data in the mutant lists"; mutant: "a helper's call is data after a call of globals or eval in the mutant lists"; mutant: "a name a subscript or attribute store changes is left as it was in the mutant lists"; mutant: "a deleted name is left as it was in the mutant lists"; mutant: "a method on a name not yet bound is passed over in the mutant lists"; mutant: "a starred item is paired with a name in the mutant lists"; mutant: "a name bound a second time keeps what it was first in the mutant lists"; mutant: "no statement before the mutant lists binds a name"; mutant: "a name bound with others in one assignment is code in the mutant lists" | +| A comparison's mutant changes the operator between the comparison's two sides, never one a string holds before it; a comparison it cannot swap, or on another line of the statement, is passed over | `tests/test_untested.py::test_a_mutant_is_made_of_an_added_line_and_never_of_a_quiet_one`, `tools/untested.py`; mutant: "a comparison is mutated where the line first shows an operator"; mutant: "a comparison it cannot swap is mutated"; mutant: "a comparison on another line of the statement is read on its first"; mutant: "a > is never swapped"; mutant: "a >= is never swapped" | +| A trailer is read from the head commit alone, never from an earlier commit of the change | `tests/test_untested.py::test_a_trailer_is_read_from_the_head_commit_alone`, `tools/untested.py`; mutant: "a trailer is read from every commit of the change"; mutant: "a trailer is read from where the checkout stands" | +| The change is read from the merge base of the base and the head, so a hunk main made since the branch left it is never the change's | `tests/test_untested.py::test_the_change_is_read_from_the_merge_base_and_not_from_main_as_it_moved_on`, `tools/untested.py`; mutant: "the change is read from the base and not from the merge base" | diff --git a/tests/test_untested.py b/tests/test_untested.py index 41cb5f26..eb9c06ad 100644 --- a/tests/test_untested.py +++ b/tests/test_untested.py @@ -91,6 +91,18 @@ def test_a_revert_no_test_notices_survives_and_one_a_test_notices_is_killed(self ("pkg/calc.py", 12, "SURVIVED", "revert")], repr(ending)) self.assertEqual(found["skipped"]["same"], 1) # the annotation + def test_the_change_is_read_from_the_merge_base_and_not_from_main_as_it_moved_on(self): + # Muse's read of #634: main changed pkg/calc.py after the branch left it. Read from main as it is, the change + # would hold main's own hunk, put back; read from the merge base, it holds the branch's alone. + made, root, first, main = change({"pkg/calc.py": CALC, "tests/test_calc.py": CALC_TEST}, + {"pkg/calc.py": CALC.replace("if a > b:", "if b < a:")}, "main moves on") + with made: + git(root, "checkout", "-q", "-b", "branch", first) + tree(root, {"pkg/calc.py": CALC.replace("x * 2", "x * 3")}) + git(root, "commit", "-q", "-am", "the branch") + found = untested.check(root, main, git(root, "rev-parse", "HEAD").strip(), float("inf"), workers=2) + self.assertEqual(outcomes(found), [("pkg/calc.py", 12, "SURVIVED", "revert")]) + def test_a_line_is_what_git_numbers_one_and_not_each_break_python_splits_on(self): label = 'LABEL = "a
b\x0cc"\n\n\ndef f():\n return %d\n' test = "import unittest\n\nfrom pkg import label\n\n\nclass L(unittest.TestCase):\n def test_f(self):\n" \ @@ -148,6 +160,17 @@ def test_a_mutant_is_made_of_an_added_line_and_never_of_a_quiet_one(self): self.assertIn(" if not (x > 1):\n", made[2]) self.assertIn(" x = x + 3\n", made[3]) self.assertIn(" return x < 3\n", made[8]) + # The comparison's own operator, not one a string holds before it (Muse's read of #634). + quoted = dict(untested.candidates("def g(x):\n return 'a < b' + str(x < 3)\n", {"new": 2, "new_n": 1})) + self.assertIn(" return 'a < b' + str(x <= 3)\n", quoted[2]) + # A comparison it cannot swap is passed over, and one on another line of the statement is never read here. + passed = dict(untested.candidates("def g(x, y):\n return x == 1 or y < 2\n", {"new": 2, "new_n": 1})) + self.assertIn(" return x == 1 or y <= 2\n", passed[2]) + spread = dict(untested.candidates("def h(y):\n return g(1,\n y < 2)\n", {"new": 2, "new_n": 2})) + self.assertIn(" return g(2,\n", spread[2]) + for now, then in (("x > 3", "x >= 3"), ("x >= 3", "x > 3")): + swapped = dict(untested.candidates("def g(x):\n return %s\n" % now, {"new": 2, "new_n": 1})) + self.assertIn(" return %s\n" % then, swapped[2], now) self.assertIn(" pass\n", dict(untested.candidates("def f(x):\n g(x)\n", {"new": 2, "new_n": 1}))[2]) annotated = "from __future__ import annotations\n\n\ndef f(x: Literal[0]) -> int:\n return x + 1\n" self.assertEqual(sorted(dict(untested.candidates(annotated, {"new": 4, "new_n": 2}))), [5]) @@ -264,19 +287,93 @@ def test_no_run_of_the_tests_begins_past_the_budget(self): self.assertEqual(probed, []) # no import is probed past the budget, the head's or an edit's def test_the_lists_of_mutants_are_data_and_the_runner_below_them_is_read(self): - text = ("ISSUE = [\n 1,\n]\nMUTANTS = [\n 2,\n # a note\n 3 if os.environ else 2,\n] + [m for m in (\n" - " (4, f(5)),\n) if m]\n\n\ndef run():\n return 4\n") + text = ("def run():\n return 4\n\n\nISSUE = [\n 1,\n]\nMUTANTS = [\n 2,\n # a note\n" + " 3 if os.environ else 2,\n] + [m for m in (\n (4, run(5)),\n) if m]\n\n\ndef main():\n" + " return run()\n") data = untested.data_lines(text) - self.assertEqual(data, {2, 5, 6, 9}) + self.assertEqual(data, {6, 9, 10, 13}) self.assertEqual(untested.data_lines("MUTANTS = [m for m in (1, 2) if True]\n"), set()) self.assertEqual(untested.data_lines("MUTANTS = [\n (1, 2),\n (3, 4)] + [\n (5, 6)]\n"), {2}) - self.assertTrue(untested.is_data({"new": 5, "new_n": 2, "old": 5, "old_n": 1}, data, data)) - self.assertTrue(untested.is_data({"new": 8, "new_n": 0, "old": 9, "old_n": 1}, data, data)) - for hunk in ({"new": 7, "new_n": 1, "old": 7, "old_n": 1}, {"new": 8, "new_n": 1, "old": 8, "old_n": 1}, - {"new": 10, "new_n": 1, "old": 10, "old_n": 1}, {"new": 14, "new_n": 1, "old": 14, "old_n": 1}, - {"new": 5, "new_n": 1, "old": 7, "old_n": 1}): + self.assertTrue(untested.is_data({"new": 9, "new_n": 2, "old": 9, "old_n": 1}, data, data)) + self.assertTrue(untested.is_data({"new": 12, "new_n": 0, "old": 13, "old_n": 1}, data, data)) + for hunk in ({"new": 11, "new_n": 1, "old": 11, "old_n": 1}, {"new": 12, "new_n": 1, "old": 12, "old_n": 1}, + {"new": 14, "new_n": 1, "old": 14, "old_n": 1}, {"new": 18, "new_n": 1, "old": 18, "old_n": 1}, + {"new": 2, "new_n": 1, "old": 2, "old_n": 1}, {"new": 9, "new_n": 1, "old": 11, "old_n": 1}): self.assertFalse(untested.is_data(hunk, data, data), hunk) + def test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers(self): + # Muse's read of #634: a call of a builtin or of another module's function may read the world, so the entry + # that holds it decides, and its hunk is run. A helper the module defines, and a text's replace, are data. + text = ("MUTANTS = [\n ('a', helper('x')),\n ('b', os.getenv('X', 'old')),\n ('c', abs(-1)),\n" + " ('d', open('x').read()),\n ('e', __import__('os').getenv('X')),\n" + " ('f', TEXT.replace('a', 'b')),\n ('g', helper(x=1)),\n ('h', ROOT.replace('x')),\n" + " ('i', os.environ['X']),\n ('j', sys.platform),\n ('k', OLD[sys.platform]),\n" + " ('l', helper('x').replace('x', 'y')),\n ('m', 'a b'.replace('a', 'c')),\n]\n") + head = ("import os\nimport sys\nimport pathlib\nTEXT = 'a'\nROOT = pathlib.Path('.')\n\n\n" + "def helper(text: str) -> str:\n return text\n\n\n") + # A text's replace is data, but ROOT's, a path's, renames a file; what is read of os or sys reads the runner. + self.assertEqual(untested.data_lines(head + text), {13, 18, 24, 25}) + # Another module's replace renames a file, as does a path's; a name bound twice may be anything, as here, where + # the list is read while helper is still os.getenv; a helper defined after the list is no helper of it. + self.assertEqual(untested.data_lines("import os\nMUTANTS = [\n ('a', os.replace('x', 'y')),\n" + " ('b', open('x').read().replace('a', 'b')),\n]\n"), set()) + shadowed = ("from os import getenv as helper\nMUTANTS = [\n ('a', helper('X')),\n]\n\n\n" + "def helper(x):\n return x\n") + self.assertEqual(untested.data_lines(shadowed), set()) + later = "MUTANTS = [\n ('a', later('X')),\n]\n\n\ndef later(x):\n return x\n" + self.assertEqual(untested.data_lines(later), set()) + # A, B = x, y binds each name to its own value, as tools/mutants.py binds the issue Kernel's path and tests. + paired = "PY, TESTS = 'kernel/issue.py', ('t',)\nMUTANTS = [\n ('a', PY.replace('k', 'K'), TESTS),\n]\n" + self.assertEqual(untested.data_lines(paired), {3}) + # Muse's and the Claude review's further cases. A walrus, a global in a helper, a method that fills a list, and + # globals() or eval may each change a name past what its assignment says; a helper's __globals__ is the whole + # module; a decorator may make a helper anything; and only a helper that says it gives a text gives one. + for case in ("import os\nTEXT = 'a'\nA = (TEXT := os.environ['X'])\nMUTANTS = [\n ('a', TEXT),\n]\n", + "import os\nVAL = [1]\nVAL.append(os.environ['X'])\nMUTANTS = [\n ('a', VAL),\n]\n", + "import os\nTEXT = 'a'\nglobals()['TEXT'] = os.environ['X']\nMUTANTS = [\n ('a', TEXT),\n]\n", + "TEXT = 'a'\nMUTANTS = [\n (eval('1'), TEXT),\n]\n", + "import os\nTEXT = 'a'\nMUTANTS = [\n (TEXT := os.environ['X'], TEXT),\n]\n", + "import os\nTEXT = 'a'\n\n\ndef helper():\n global TEXT\n TEXT = os.environ['X']\n" + " return 1\n\n\nMUTANTS = [\n ('b', TEXT),\n]\n", + "import os\nVAL = [1]\n\n\ndef helper(x):\n VAL.append(x)\n return x\n\n\n" + "MUTANTS = [\n ('b', VAL),\n]\n", + "import os\n\n\ndef helper(x):\n return x\n\n\nMUTANTS = [\n" + " ('a', helper.__globals__['os'].environ['X']),\n]\n", + "import os\n\n\n@lambda f: os.getenv\ndef helper(x):\n return x\n\n\nMUTANTS = [\n" + " ('a', helper('X')),\n]\n", + "import pathlib\nROOT = pathlib.Path('.')\n\n\ndef helper():\n return ROOT\n\n\nMUTANTS = [\n" + " ('a', helper().replace('x')),\n]\n", + "import os\nMUTANTS = [\n ('a', os.environ['X'].replace('a', 'b').replace('c', 'd')),\n]\n", + "import os\n\n\ndef helper(x):\n return x\n\n\nglobals()['helper'] = os.getenv\nMUTANTS = [\n" + " ('a', helper('X')),\n]\n", + "import os\nVAL = [1]\n\n\ndef helper(x):\n VAL[0] = os.environ['X']\n return x\n\n\n" + "MUTANTS = [\n ('b', VAL),\n]\n", + "def open(x) -> str:\n return x\n\n\ndel open\nMUTANTS = [\n ('a', open('README.md')),\n]\n", + "import os\n\n\ndef helper(x):\n VAL.append(os.environ['X'])\n return x\n\n\nVAL = [1]\n" + "MUTANTS = [\n ('b', VAL),\n]\n", + "import os\nEMPTY = ()\nA, B = *EMPTY, *(os.getcwd(), os.getcwd())\nMUTANTS = [\n ('a', A),\n]\n", + "TEXT = 'a'\nMUTANTS = [\n ('a', TEXT.format(1)),\n]\n"): + self.assertEqual(untested.data_lines(case), set(), case) + # A helper's own name, an annotated text, and a method on a text are no change to what a name is. + kept = ("TEXT: str = 'a'\nPAIR = ('a', 'b')\n\n\ndef helper(text: str, PAIR=()) -> str:\n text.upper()\n" + " PAIR.index('a')\n return text\n\n\nMUTANTS = [\n ('a', TEXT.replace('a', 'b')),\n" + " ('b', helper('x').replace('x', 'y')),\n ('c', PAIR),\n]\n") + self.assertEqual(untested.data_lines(kept), {12, 13, 14}) + later = ("def helper(x: str) -> str:\n return LATER.replace('a', x)\n\n\nLATER = 'a'\nMUTANTS = [\n" + " ('a', LATER.replace('a', 'b')),\n]\n") + self.assertEqual(untested.data_lines(later), {7}) + # An import behind an if, a try or a with binds its name as one at the top does; a star import binds names no + # one can read, so a name nothing bound is code; and so is a name bound to what code gave, or by an unpacking. + for fenced in ("if True:\n import os\n", "try:\n import os\nexcept ImportError:\n os = None\n", + "with x:\n import os.path\n", "import os.path\n", "from os import *\n", + "import os\nENV = os.environ\n", "import os\nA, B = os.environ['X'].split()\n", + "import pathlib\nENV = 'a'\nfor ENV in [pathlib.Path('.')]:\n pass\n", + "class ENV:\n pass\n", "os = 'a'\nif True:\n import os\n", "os = 'a'\nimport os.path\n", + "ENV = 'a'\nENV, B = os.environ['X'].split()\n"): + for read in ("os.environ['X']", "environ['X']", "ENV['X']", "A", "ENV.replace('x')"): + text = fenced + "MUTANTS = [\n ('a', %s),\n]\n" % read + self.assertEqual(untested.data_lines(text), set(), text) + CRASHED = ("=" * 70 + "\nERROR: test_x (tests.test_calc.Calc)\n" + "-" * 70 + "\nTraceback (most recent call last):\n" " File \"x.py\", line 1, in test_x\nNameError: name 'pick' is not defined\n\n" + "-" * 70 + @@ -332,6 +429,19 @@ def test_the_head_commits_trailer_covers_its_survivor(self): "Reworded\n\nBehaviour-preserving: pkg/calc.py:5-5\n") self.assertEqual((reworded["hunks"], reworded["problems"]), ([], [])) + def test_a_trailer_is_read_from_the_head_commit_alone(self): + # Muse's read of #634: lines move each fix round, so a trailer an earlier commit of the change wrote names lines + # that may no longer be the ones it meant, and is never read. + made, root, first, earlier = change({"pkg/calc.py": CALC}, {"pkg/calc.py": CALC.replace("x * 2", "x + x")}, + "Faster\n\nBehaviour-preserving: pkg/calc.py:12-12\n") + with made: + self.assertEqual(untested.trailers(root, earlier), ["pkg/calc.py:12-12"]) + tree(root, {"pkg/notes.py": "A = 1\n"}) + git(root, "add", "-A") + git(root, "commit", "-q", "-m", "A fix round\n\nIt says nothing of the lines.\n") + self.assertEqual(untested.trailers(root, git(root, "rev-parse", "HEAD").strip()), []) + self.assertEqual(untested.trailers(root, earlier), ["pkg/calc.py:12-12"]) # the commit asked, not HEAD + class FirstTests(unittest.TestCase): def setUp(self): diff --git a/tools/mutants.py b/tools/mutants.py index f8a71038..5fe22a33 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -15381,9 +15381,126 @@ def moved_before(rel: str, block: str, anchor: str) -> tuple: " return bool(new | old) and new <= head_lines and old <= base_lines\n", " return bool(new | old)\n", ("Reverts.test_the_lists_of_mutants_are_data_and_the_runner_below_them_is_read",)), ("an entry of the mutant lists that decides is data", "tools/untested.py", - " if isinstance(inner, (ast.List, ast.Tuple)) and child in inner.elts and _literal(child):\n", + " if isinstance(inner, (ast.List, ast.Tuple)) and child in inner.elts and _literal(child, bound):\n", " if isinstance(inner, (ast.List, ast.Tuple)) and child in inner.elts:\n", ("Reverts.test_the_lists_of_mutants_are_data_and_the_runner_below_them_is_read",)), + # Muse's read of #634: a call in the mutant lists is data only of the module's own helpers, or a text's replace. + ("a call of a builtin or another module's function in the mutant lists is data", "tools/untested.py", + ' and bound.get(func.id) in ("helper", "text helper")) or (', + " ) or (", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a call of any method in the mutant lists is data", "tools/untested.py", + 'isinstance(func, ast.Attribute) and func.attr == "replace" and _text(func.value, bound))', + 'isinstance(func, ast.Attribute) and _text(func.value, bound))', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a comparison it cannot swap is mutated", "tools/untested.py", + " and type(inner.ops[0]) in SWAP):\n", " ):\n", + ("Reverts.test_a_mutant_is_made_of_an_added_line_and_never_of_a_quiet_one",)), + ("a comparison on another line of the statement is read on its first", "tools/untested.py", + " if (isinstance(inner, ast.Compare) and inner.lineno == inner.end_lineno == node.lineno\n", + " if (isinstance(inner, ast.Compare)\n", + ("Reverts.test_a_mutant_is_made_of_an_added_line_and_never_of_a_quiet_one",)), + ("a name bound a second time keeps what it was first in the mutant lists", "tools/untested.py", + ' bound[name] = "code" if name in bound else kind\n', " bound[name] = kind\n", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("no statement before the mutant lists binds a name", "tools/untested.py", + " if not listed:\n _bind(bound, node)\n continue\n", + " if not listed:\n continue\n", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a name bound with others in one assignment is code in the mutant lists", "tools/untested.py", + " list(zip(target.elts, node.value.elts)) if isinstance(target, ast.Tuple)\n", + " [] if isinstance(target, ast.Tuple)\n", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a name an import in another statement binds is data", "tools/untested.py", + " elif isinstance(inner, (ast.Import, ast.ImportFrom)):\n", + " elif False:\n", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a dotted import binds its whole path", "tools/untested.py", + ' named += [(alias.asname or alias.name.split(".")[0], "code") for alias in inner.names]\n', + ' named += [(alias.asname or alias.name, "code") for alias in inner.names]\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a name another unpacking binds is data", "tools/untested.py", + ' if not pairs:\n named += [(inner.id, "code") for inner in ast.walk(target) if isinstance(inner, ast.Name)]\n', + "", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a value read from code is data", "tools/untested.py", + ' named.append((name.id, "value" if _literal(value, bound) else "code"))\n', + ' named.append((name.id, "value"))\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a walrus leaves a name as it was in the mutant lists", "tools/untested.py", + ' if isinstance(inner, ast.NamedExpr):\n', ' if False:\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a global in a helper leaves a name as it was in the mutant lists", "tools/untested.py", + ' elif isinstance(inner, (ast.Global, ast.Nonlocal)):\n', ' elif False:\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a call of globals or eval leaves every name as it was in the mutant lists", "tools/untested.py", + ' bound["*"] = "code"\n', ' pass\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("every name is read as it was after a call of globals or eval in the mutant lists", "tools/untested.py", + ' return bound.get("*") != "code" and bound.get(node.id) in ("text", "value", "helper", "text helper")\n', ' return bound.get(node.id) in ("text", "value", "helper", "text helper")\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a method called on a name leaves it as it was in the mutant lists", "tools/untested.py", + ' changed = name\n', ' pass\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a method a function calls on its own name changes the module's in the mutant lists", "tools/untested.py", + ' if changed is not None and changed not in own:\n', ' if changed is not None:\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a decorated helper is a helper in the mutant lists", "tools/untested.py", + ' named = [(node.name, "code" if node.decorator_list else "text helper" if gives else "helper")]\n', ' named = [(node.name, "text helper" if gives else "helper")]\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a dunder attribute is data in the mutant lists", "tools/untested.py", + ' if isinstance(node, ast.Attribute) and node.attr.startswith("__"):\n', ' if False:\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("any helper gives a text in the mutant lists", "tools/untested.py", + 'isinstance(node.func, ast.Name) and bound.get(node.func.id) == "text helper")', 'isinstance(node.func, ast.Name) and bound.get(node.func.id) in ("helper", "text helper"))', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("an annotated assignment is code in the mutant lists", "tools/untested.py", + ' elif isinstance(node, (ast.Assign, ast.AnnAssign)) and node.value is not None:\n', ' elif isinstance(node, ast.Assign):\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a call whose result is read as a text need not itself be data in the mutant lists", "tools/untested.py", + ' if not (isinstance(node, ast.Call) and _literal(node, bound)):\n', ' if not isinstance(node, ast.Call):\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a helper's call is data after a call of globals or eval in the mutant lists", "tools/untested.py", + ' named = (isinstance(func, ast.Name) and bound.get("*") != "code"\n', ' named = (isinstance(func, ast.Name)\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a name a subscript or attribute store changes is left as it was in the mutant lists", "tools/untested.py", + " changed = inner.value.id # X[0] = ...", " pass # X[0] = ...", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a deleted name is left as it was in the mutant lists", "tools/untested.py", + " changed = inner.id # del X", " pass # del X", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a method on a name not yet bound is passed over in the mutant lists", "tools/untested.py", + "and not (name not in bound and inner.func.attr in TEXT_METHODS):\n", "and name in bound:\n", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a starred item is paired with a name in the mutant lists", "tools/untested.py", + " and not any(isinstance(e, ast.Starred) for e in target.elts + node.value.elts) else [])", + " else [])", + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a > is never swapped", "tools/untested.py", 'ast.Gt: (">", ">="), ', 'ast.Gt: (">", ">"), ', + ("Reverts.test_a_mutant_is_made_of_an_added_line_and_never_of_a_quiet_one",)), + ("a >= is never swapped", "tools/untested.py", 'ast.GtE: (">=", ">")}', 'ast.GtE: (">=", ">=")}', + ("Reverts.test_a_mutant_is_made_of_an_added_line_and_never_of_a_quiet_one",)), + ("a trailer is read from where the checkout stands", "tools/untested.py", + '",valueonly,unfold)", head)', '",valueonly,unfold)", "HEAD")', + ("Trailers.test_a_trailer_is_read_from_the_head_commit_alone",)), + ("a replace on anything is data in the mutant lists", "tools/untested.py", + 'isinstance(func, ast.Attribute) and func.attr == "replace" and _text(func.value, bound))', + 'isinstance(func, ast.Attribute) and func.attr == "replace" and _literal(func.value, bound))', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a name no statement binds before the mutant lists is data", "tools/untested.py", + ' return bound.get("*") != "code" and bound.get(node.id) in ("text", "value", "helper", "text helper")\n', + ' return bound.get("*") != "code" and bound.get(node.id) != "code"\n', + ("Reverts.test_a_call_in_the_mutant_lists_is_data_only_of_the_modules_own_helpers",)), + ("a comparison is mutated where the line first shows an operator", "tools/untested.py", + " start, end = inner.left.end_col_offset, inner.comparators[0].col_offset\n", + " start, end = 0, len(raw)\n", + ("Reverts.test_a_mutant_is_made_of_an_added_line_and_never_of_a_quiet_one",)), + ("a trailer is read from every commit of the change", "tools/untested.py", + ' said = git(root, "log", "-1", "--format=%(trailers:key="', ' said = git(root, "log", "--format=%(trailers:key="', + ("Trailers.test_a_trailer_is_read_from_the_head_commit_alone",)), + ("the change is read from the base and not from the merge base", "tools/untested.py", + ' since = git(root, "merge-base", base, head).strip()\n', ' since = base\n', + ("Reverts.test_the_change_is_read_from_the_merge_base_and_not_from_main_as_it_moved_on",)), ("a line of code beside an entry of the mutant lists is data", "tools/untested.py", " return found - code\n", " return found\n", ("Reverts.test_the_lists_of_mutants_are_data_and_the_runner_below_them_is_read",)), diff --git a/tools/untested.py b/tools/untested.py index 21824a96..0a72c32d 100644 --- a/tools/untested.py +++ b/tools/untested.py @@ -159,9 +159,13 @@ def data_lines(text: str) -> set: except SyntaxError: return set() lines, found, code = source_lines(text), set(), set() + bound = {} for node in tree.body: - if not (isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id in DATA[1] - for target in node.targets)): + listed = isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id in DATA[1] + for target in node.targets) + _unsettle(bound, node) + if not listed: + _bind(bound, node) continue found.update(at for at in range(node.lineno, node.end_lineno + 1) if not lines[at - 1].strip() or lines[at - 1].strip().startswith("#")) @@ -171,32 +175,134 @@ def data_lines(text: str) -> set: if hasattr(inner, "lineno"): code.update((inner.lineno, inner.end_lineno)) for child in ast.iter_child_nodes(inner): - if isinstance(inner, (ast.List, ast.Tuple)) and child in inner.elts and _literal(child): + if isinstance(inner, (ast.List, ast.Tuple)) and child in inner.elts and _literal(child, bound): found.update(range(child.lineno, child.end_lineno + 1)) else: todo.append(child) + _bind(bound, node) return found - code -def _literal(node) -> bool: +def _bind(bound: dict, node) -> None: + """Read into `bound` what the statement `node`, at the module's top, binds each name to, as the module runs in + order: "helper" for a function it defines, "text" for a text it gives, "value" for a value `_literal` takes, and + "code" for anything else any statement binds, an import's name wherever it stands among them, and for a name bound a + second time, which may then be anything. A name no statement binds before the list is in none, and is code.""" + named = [] + if isinstance(node, ast.FunctionDef): + # A decorator may make the name anything; a helper that says it gives a text is taken at its word. + gives = isinstance(node.returns, ast.Name) and node.returns.id == "str" + named = [(node.name, "code" if node.decorator_list else "text helper" if gives else "helper")] + elif isinstance(node, (ast.Assign, ast.AnnAssign)) and node.value is not None: + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + # A, B = x, y binds each name to its own value; any other unpacking may bind anything. + pairs = [(target, node.value)] if isinstance(target, ast.Name) else ( + list(zip(target.elts, node.value.elts)) if isinstance(target, ast.Tuple) + and isinstance(node.value, ast.Tuple) and len(target.elts) == len(node.value.elts) + and not any(isinstance(e, ast.Starred) for e in target.elts + node.value.elts) else []) + for name, value in pairs: + if not isinstance(name, ast.Name): + named += [(inner.id, "code") for inner in ast.walk(name) if isinstance(inner, ast.Name)] + elif isinstance(value, ast.Constant) and isinstance(value.value, str): + named.append((name.id, "text")) + else: + named.append((name.id, "value" if _literal(value, bound) else "code")) + if not pairs: + named += [(inner.id, "code") for inner in ast.walk(target) if isinstance(inner, ast.Name)] + else: + for inner in ast.walk(node): + if isinstance(inner, ast.Name) and isinstance(inner.ctx, ast.Store): + named.append((inner.id, "code")) + elif isinstance(inner, (ast.Import, ast.ImportFrom)): + named += [(alias.asname or alias.name.split(".")[0], "code") for alias in inner.names] + elif isinstance(inner, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + named.append((inner.name, "code")) + for name, kind in named: + bound[name] = "code" if name in bound else kind + + +REFLECTS = ("globals", "locals", "vars", "exec", "eval", "setattr", "delattr") +TEXT_METHODS = ("replace", "format", "join", "split", "strip", "startswith", "endswith", "upper", "lower", "index", + "count", "find", "splitlines", "partition") + + +def _unsettle(bound: dict, node) -> None: + """Make code in `bound` each name the statement `node` may bind or change other than as an assignment's target: by + a walrus anywhere in it, a `global` or `nonlocal` in a function it defines, or a method called on it, at the top or + in a function where it is not the function's own; a text alone, which no method changes, is left as it is. A call + of `globals`, `exec` or the like may bind any name, so after one every name is code.""" + for inner in ast.walk(node): + if isinstance(inner, ast.NamedExpr): + bound[inner.target.id] = "code" + elif isinstance(inner, (ast.Global, ast.Nonlocal)): + bound.update((name, "code") for name in inner.names) + elif isinstance(inner, ast.Call) and isinstance(inner.func, ast.Name) and inner.func.id in REFLECTS: + bound["*"] = "code" + own = set() + for inner in ast.walk(node): + if isinstance(inner, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + own.update(arg.arg for arg in ast.walk(inner.args) if isinstance(arg, ast.arg)) + own.update(name.id for name in ast.walk(inner) if isinstance(name, ast.Name) and isinstance(name.ctx, ast.Store)) + for inner in ast.walk(node): + changed = None + if isinstance(inner, ast.Call) and isinstance(inner.func, ast.Attribute) and isinstance(inner.func.value, ast.Name): + # A method on a text changes nothing; on a name not bound yet, only one no text has may change what it + # will be, as a helper defined before its list fills it when called. + name = inner.func.value.id + if bound.get(name) != "text" and not (name not in bound and inner.func.attr in TEXT_METHODS): + changed = name + elif (isinstance(inner, (ast.Subscript, ast.Attribute)) and isinstance(inner.ctx, (ast.Store, ast.Del)) + and isinstance(inner.value, ast.Name)): + changed = inner.value.id # X[0] = ..., del X[0], X.y = ..., in a function or at the top + elif isinstance(inner, ast.Name) and isinstance(inner.ctx, ast.Del): + changed = inner.id # del X: it may then be a builtin + if changed is not None and changed not in own: + bound[changed] = "code" + + +def _literal(node, bound: dict) -> bool: """Whether `node` is made of constants, names, attributes and items alone, in tuples, lists, sums, products, and - calls of such a name or attribute given no keyword, each maybe unpacked: a mutant's entry, as its helpers build one, - and nothing that decides.""" - if isinstance(node, (ast.Constant, ast.Name)): + calls given no keyword of a helper the module defines, or of `replace` on a text, each maybe unpacked: a mutant's + entry, as its helpers build one, and nothing that decides. A builtin's call or another module's, and anything read + from a name an import binds, as `os.environ` or `sys.platform`, may read the world, so it is code (decision + 0166). `bound` is what `_bind` has read of the module before the list.""" + if isinstance(node, ast.Constant): return True + if isinstance(node, ast.Name): + return bound.get("*") != "code" and bound.get(node.id) in ("text", "value", "helper", "text helper") + if isinstance(node, ast.Attribute) and node.attr.startswith("__"): + return False # a function's __globals__, a class's __dict__: the module's every name, read if isinstance(node, (ast.Attribute, ast.Starred)): - return _literal(node.value) + return _literal(node.value, bound) if isinstance(node, ast.Subscript): - return _literal(node.value) and _literal(node.slice) + return _literal(node.value, bound) and _literal(node.slice, bound) if isinstance(node, (ast.Tuple, ast.List)): - return all(_literal(item) for item in node.elts) + return all(_literal(item, bound) for item in node.elts) if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.Add, ast.Mult)): - return _literal(node.left) and _literal(node.right) + return _literal(node.left, bound) and _literal(node.right, bound) if isinstance(node, ast.Call): - return _literal(node.func) and not node.keywords and all(_literal(arg) for arg in node.args) + func = node.func + named = (isinstance(func, ast.Name) and bound.get("*") != "code" + and bound.get(func.id) in ("helper", "text helper")) or ( + isinstance(func, ast.Attribute) and func.attr == "replace" and _text(func.value, bound)) + return named and not node.keywords and all(_literal(arg, bound) for arg in node.args) return False +def _text(node, bound: dict) -> bool: + """Whether `node` is a text: a constant one, a name the module gives one, or what a helper that says it gives a text, + or a text's `replace`, gives; any other helper may give a path, whose `replace` renames a file.""" + if isinstance(node, ast.Constant): + return isinstance(node.value, str) + if isinstance(node, ast.Name): + return bound.get("*") != "code" and bound.get(node.id) == "text" + if not (isinstance(node, ast.Call) and _literal(node, bound)): + return False + return (isinstance(node.func, ast.Name) and bound.get(node.func.id) == "text helper") or ( + isinstance(node.func, ast.Attribute) and node.func.attr == "replace") + + def is_data(hunk: dict, head_lines: set, base_lines: set) -> bool: """Whether every line `hunk` adds is data at the head, and every line it removes was data at the base.""" new = set(range(hunk["new"], hunk["new"] + hunk["new_n"])) @@ -476,8 +582,7 @@ def _quiet(node) -> bool: return False -COMPARE = re.compile(r" (<=|>=|<|>) ") -SWAP = {"<": "<=", "<=": "<", ">": ">=", ">=": ">"} +SWAP = {ast.Lt: ("<", "<="), ast.LtE: ("<=", "<"), ast.Gt: (">", ">="), ast.GtE: (">=", ">")} def _mutated(node, line: str): @@ -488,9 +593,16 @@ def _mutated(node, line: str): raw = line.encode() # the offsets are of UTF-8 bytes return (raw[:test.col_offset] + b"not (" + raw[test.col_offset:test.end_col_offset] + b")" + raw[test.end_col_offset:]).decode() - compare = COMPARE.search(line) - if compare and any(isinstance(inner, ast.Compare) for inner in ast.walk(node)): - return line[:compare.start()] + " %s " % SWAP[compare.group(1)] + line[compare.end():] + for inner in ast.walk(node): + # The operator is found between the comparison's two sides, never where the line first shows one: in a string + # it would be no comparison, and a test that reads the text would kill a mutant of nothing (Muse, #634). + if (isinstance(inner, ast.Compare) and inner.lineno == inner.end_lineno == node.lineno + and type(inner.ops[0]) in SWAP): + raw = line.encode() + start, end = inner.left.end_col_offset, inner.comparators[0].col_offset + now, new = SWAP[type(inner.ops[0])] + gap = raw[start:end].decode() + return raw[:start].decode() + gap.replace(now, new, 1) + raw[end:].decode() for inner in ast.walk(node): if isinstance(inner, ast.Constant) and inner.lineno == node.lineno and inner.end_lineno == node.lineno: if isinstance(inner.value, bool) or (isinstance(inner.value, int) and not isinstance(inner.value, bool)):