From eff2a07d17078f8fb0470622a154be76665de470 Mon Sep 17 00:00:00 2001 From: KennyMcSimpson <70093673+KennyMcSimpson@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:15:05 +0800 Subject: [PATCH] Use UTF-8 for JSON command-line file I/O --- doc/commandline.rst | 3 ++ jsonpatch_cli.py | 12 ++++---- tests.py | 68 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/doc/commandline.rst b/doc/commandline.rst index a7a78d8..e532ed7 100644 --- a/doc/commandline.rst +++ b/doc/commandline.rst @@ -4,6 +4,9 @@ Commandline Utilities The JSON patch package contains the commandline utilities ``jsondiff`` and ``jsonpatch``. +Both utilities read JSON files as UTF-8. When ``jsonpatch`` modifies a file +in-place, it also writes UTF-8. + ``jsondiff`` ------------ diff --git a/jsonpatch_cli.py b/jsonpatch_cli.py index ab605fc..e84d0f7 100644 --- a/jsonpatch_cli.py +++ b/jsonpatch_cli.py @@ -12,8 +12,8 @@ jsondiff_parser = argparse.ArgumentParser( prog='jsondiff', description='Diff two JSON files') -jsondiff_parser.add_argument('FILE1', type=argparse.FileType('r')) -jsondiff_parser.add_argument('FILE2', type=argparse.FileType('r')) +jsondiff_parser.add_argument('FILE1', type=argparse.FileType('r', encoding='utf-8')) +jsondiff_parser.add_argument('FILE2', type=argparse.FileType('r', encoding='utf-8')) jsondiff_parser.add_argument('--indent', type=int, default=None, help='Indent output by n spaces') jsondiff_parser.add_argument('-u', '--preserve-unicode', action='store_true', @@ -43,9 +43,9 @@ def diff_files(): jsonpatch_parser = argparse.ArgumentParser( prog='jsonpatch', description='Apply a JSON patch on a JSON file') -jsonpatch_parser.add_argument('ORIGINAL', type=argparse.FileType('r'), +jsonpatch_parser.add_argument('ORIGINAL', type=argparse.FileType('r', encoding='utf-8'), help='Original file') -jsonpatch_parser.add_argument('PATCH', type=argparse.FileType('r'), +jsonpatch_parser.add_argument('PATCH', type=argparse.FileType('r', encoding='utf-8'), nargs='?', default=sys.stdin, help='Patch file (read from stdin if omitted)') jsonpatch_parser.add_argument('--indent', type=int, default=None, @@ -86,7 +86,7 @@ def patch_files(): # because atomic renames do not work across mount points.) fd, pathname = tempfile.mkstemp(dir=dirname) - fp = os.fdopen(fd, 'w') + fp = os.fdopen(fd, 'w', encoding='utf-8') atomic = True except OSError: @@ -96,7 +96,7 @@ def patch_files(): if args.backup: os.rename(args.ORIGINAL.name, args.ORIGINAL.name + '.orig') - fp = open(args.ORIGINAL.name, 'w') + fp = open(args.ORIGINAL.name, 'w', encoding='utf-8') atomic = False else: diff --git a/tests.py b/tests.py index 124b086..8209baf 100755 --- a/tests.py +++ b/tests.py @@ -1836,28 +1836,35 @@ def setUp(self): self.addCleanup(tmp.cleanup) self.dir = tmp.name - def write(self, name, value): + def write(self, name, value, ensure_ascii=True): path = os.path.join(self.dir, name) with open(path, 'w', encoding='utf-8') as f: - json.dump(value, f) + json.dump(value, f, ensure_ascii=ensure_ascii) return path def read(self, path): with open(path, encoding='utf-8') as f: return json.load(f) - def run_script(self, script, *args, stdin=''): + def run_python(self, code, *args, stdin='', utf8=True): here = os.path.dirname(os.path.abspath(__file__)) # the scripts have to import this jsonpatch, not an installed one pythonpath = [here] + [p for p in [os.environ.get('PYTHONPATH')] if p] env = dict(os.environ, PYTHONPATH=os.pathsep.join(pythonpath), PYTHONUTF8='1') + if not utf8: + # Exercise locale-dependent file I/O while keeping pipes in UTF-8. + env.update(PYTHONUTF8='0', PYTHONIOENCODING='utf-8', + LC_ALL='C', PYTHONCOERCECLOCALE='0') return subprocess.run( - [sys.executable, '-c', - 'import jsonpatch_cli; jsonpatch_cli.{0}_main()'.format(script)] - + list(args), + [sys.executable, '-c', code] + list(args), input=stdin, capture_output=True, encoding='utf-8', env=env) + def run_script(self, script, *args, stdin='', utf8=True): + return self.run_python( + 'import jsonpatch_cli; jsonpatch_cli.{0}_main()'.format(script), + *args, stdin=stdin, utf8=utf8) + def test_jsondiff(self): src = self.write('a.json', {'a': [1, 2], 'b': 0}) dst = self.write('b.json', {'a': [1, 2, 3], 'c': 100}) @@ -1887,6 +1894,18 @@ def test_jsondiff_preserve_unicode(self): self.assertIn('"\\u00e4"', self.run_script('jsondiff', src, dst).stdout) self.assertIn('"รค"', self.run_script('jsondiff', '-u', src, dst).stdout) + def test_jsondiff_utf8_files_without_utf8_mode(self): + src = self.write('a.json', {'a': 'old', 'unchanged': 'รค'}, + ensure_ascii=False) + dst = self.write('b.json', {'a': '๐Ÿš€', 'unchanged': 'รค'}, + ensure_ascii=False) + res = self.run_script('jsondiff', src, dst, utf8=False) + self.assertEqual(res.returncode, 1, res.stderr) + patch = json.loads(res.stdout) + self.assertEqual(patch, [{'op': 'replace', 'path': '/a', 'value': '๐Ÿš€'}]) + self.assertEqual(jsonpatch.apply_patch(self.read(src), patch), + self.read(dst)) + def test_jsonpatch(self): doc = self.write('doc.json', {'a': [1, 2], 'b': 0}) patch = self.write('patch.json', [{'op': 'add', 'path': '/a/-', 'value': 3}, @@ -1903,6 +1922,17 @@ def test_jsonpatch_patch_from_stdin(self): self.assertEqual(res.returncode, 0, res.stderr) self.assertEqual(json.loads(res.stdout), {'a': 2}) + def test_jsonpatch_utf8_files_without_utf8_mode(self): + doc = self.write('doc.json', {'a': 'รค'}, ensure_ascii=False) + patch = self.write('patch.json', [ + {'op': 'test', 'path': '/a', 'value': 'รค'}, + {'op': 'replace', 'path': '/a', 'value': '๐Ÿš€'}, + ], ensure_ascii=False) + res = self.run_script('jsonpatch', doc, patch, utf8=False) + self.assertEqual(res.returncode, 0, res.stderr) + self.assertEqual(json.loads(res.stdout), {'a': '๐Ÿš€'}) + self.assertEqual(self.read(doc), {'a': 'รค'}) + def test_jsonpatch_indent_preserve_unicode(self): doc = self.write('doc.json', {'a': 'x'}) patch = self.write('patch.json', [{'op': 'replace', 'path': '/a', 'value': 'รค'}]) @@ -1920,6 +1950,28 @@ def test_jsonpatch_in_place(self): # the temporary file has been moved over the original self.assertEqual(sorted(os.listdir(self.dir)), ['doc.json', 'patch.json']) + def test_jsonpatch_in_place_utf8_without_utf8_mode(self): + for atomic in (True, False): + with self.subTest(atomic=atomic): + doc = self.write('doc.json', {'a': 'old'}) + patch = self.write('patch.json', [ + {'op': 'replace', 'path': '/a', 'value': '๐Ÿš€'}, + ]) + if atomic: + code = 'import jsonpatch_cli; jsonpatch_cli.jsonpatch_main()' + else: + code = ( + 'from unittest import mock\n' + 'import jsonpatch_cli\n' + 'with mock.patch("tempfile.mkstemp", side_effect=OSError):\n' + ' jsonpatch_cli.jsonpatch_main()\n' + ) + res = self.run_python(code, '-i', '-u', doc, patch, utf8=False) + self.assertEqual((res.returncode, res.stdout), (0, ''), res.stderr) + self.assertEqual(self.read(doc), {'a': '๐Ÿš€'}) + self.assertEqual(sorted(os.listdir(self.dir)), + ['doc.json', 'patch.json']) + # 2026-10-11: Windows cannot replace the original while its input is open. def test_jsonpatch_in_place_closes_original_before_replace(self): doc = self.write('doc.json', {'a': 1}) @@ -1971,8 +2023,8 @@ def test_jsonpatch_in_place_cleans_temp_on_output_error(self): opened = [] fdopen = os.fdopen - def open_output(fd, mode): - stream = fdopen(fd, mode) + def open_output(fd, mode, **kwargs): + stream = fdopen(fd, mode, **kwargs) self.addCleanup(stream.close) opened.append(stream) proxy = mock.Mock(wraps=stream)