fix(kvb-org-operator): keep UTF-8 input whole at every prompt
operator.py run crashed with UnicodeDecodeError when the operator erased an umlaut on a terminal without IUTF8: the erase key removed only the second byte of the character and left a lone 0xc3 for input() to decode strictly. typed() and timed() now set IUTF8 on a terminal stdin while they wait for a line and restore the previous mode afterwards, and run replaces undecodable bytes instead of crashing. 🎯 Fix: 95 % 💥 Break: 2 % 💸 Tokens: 100 % ⏱️ Velocity: 100 % 🛡️ Security: ~100 % 🏆 Score: 93 🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a2a03995cb
commit
2ed3628d9b
2 files changed
+119
No files matched your search
@@ -136,6 +136,7 @@ import contextlib
|
|||||||
import errno
|
import errno
|
||||||
import hashlib
|
import hashlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import io
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import select
|
import select
|
||||||
@@ -521,6 +522,24 @@ def ask(holdable: bool, read: Callable[[str], str | None]) -> str:
|
|||||||
return answer.strip().lower()
|
return answer.strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def whole_characters(read: Callable[[str], str | None]) -> Callable[[str], str | None]:
|
||||||
|
"""Returns:
|
||||||
|
read with IUTF8 set on a terminal stdin while it waits for a line and
|
||||||
|
the previous terminal mode restored afterwards.
|
||||||
|
"""
|
||||||
|
def reading(prompt: str) -> str | None:
|
||||||
|
if not sys.stdin.isatty():
|
||||||
|
return read(prompt)
|
||||||
|
mode = termios.tcgetattr(sys.stdin.fileno())
|
||||||
|
termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, [mode[0] | termios.IUTF8, *mode[1:]])
|
||||||
|
try:
|
||||||
|
return read(prompt)
|
||||||
|
finally:
|
||||||
|
termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, mode)
|
||||||
|
return reading
|
||||||
|
|
||||||
|
|
||||||
|
@whole_characters
|
||||||
def typed(prompt: str) -> str | None:
|
def typed(prompt: str) -> str | None:
|
||||||
try:
|
try:
|
||||||
return input(prompt)
|
return input(prompt)
|
||||||
@@ -528,6 +547,7 @@ def typed(prompt: str) -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@whole_characters
|
||||||
def timed(prompt: str) -> str | None:
|
def timed(prompt: str) -> str | None:
|
||||||
"""Returns:
|
"""Returns:
|
||||||
One line read unbuffered from stdin, None at its end or when no line
|
One line read unbuffered from stdin, None at its end or when no line
|
||||||
@@ -899,6 +919,8 @@ def cmd_run(rasci: ModuleType, args: argparse.Namespace) -> int:
|
|||||||
shell = os.environ.get("SHELL")
|
shell = os.environ.get("SHELL")
|
||||||
if not shell:
|
if not shell:
|
||||||
fail("SHELL is not set; run executes each command in the operator's own shell")
|
fail("SHELL is not set; run executes each command in the operator's own shell")
|
||||||
|
if isinstance(sys.stdin, io.TextIOWrapper):
|
||||||
|
sys.stdin.reconfigure(errors="replace")
|
||||||
steps = [row for row in open_rows(rasci, None) if runnable(row)]
|
steps = [row for row in open_rows(rasci, None) if runnable(row)]
|
||||||
if not steps:
|
if not steps:
|
||||||
print("operator: nothing to run")
|
print("operator: nothing to run")
|
||||||
|
|||||||
@@ -5,10 +5,13 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import pty
|
import pty
|
||||||
|
import select
|
||||||
import shutil
|
import shutil
|
||||||
|
import signal
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import termios
|
||||||
import time
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -121,6 +124,60 @@ class TestOperator(unittest.TestCase):
|
|||||||
chunks.append(chunk)
|
chunks.append(chunk)
|
||||||
return b"".join(chunks).decode()
|
return b"".join(chunks).decode()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _chunk(leader: int) -> bytes:
|
||||||
|
if not select.select([leader], [], [], 30)[0]:
|
||||||
|
return b""
|
||||||
|
try:
|
||||||
|
return os.read(leader, 65536)
|
||||||
|
except OSError:
|
||||||
|
return b""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _iutf8(leader: int) -> bool:
|
||||||
|
return bool(termios.tcgetattr(leader)[0] & termios.IUTF8)
|
||||||
|
|
||||||
|
def _terminal(self, args: list[str], steps: list[tuple[bytes, bytes | signal.Signals]]) -> tuple[int, str, list[bool]]:
|
||||||
|
"""Args:
|
||||||
|
args: the operator.py arguments.
|
||||||
|
steps: each prompt to wait for and what the operator does at it:
|
||||||
|
the keys typed, or a signal sent once operator.py waits in
|
||||||
|
the read.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The exit code of operator.py on a terminal without IUTF8 under a
|
||||||
|
strict UTF-8 stdin, everything the terminal showed, and whether
|
||||||
|
IUTF8 was set at each prompt reached and after the exit.
|
||||||
|
"""
|
||||||
|
leader, follower = pty.openpty()
|
||||||
|
self.addCleanup(os.close, leader)
|
||||||
|
mode = termios.tcgetattr(follower)
|
||||||
|
termios.tcsetattr(follower, termios.TCSANOW, [mode[0] & ~termios.IUTF8, *mode[1:]])
|
||||||
|
try:
|
||||||
|
child = subprocess.Popen(
|
||||||
|
[sys.executable, str(OPERATOR), *args], stdin=follower, stdout=follower, stderr=follower,
|
||||||
|
env={**self.env, "SHELL": "/bin/sh", "PYTHONIOENCODING": "utf-8:strict"}, cwd=self.tmp.name,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
os.close(follower)
|
||||||
|
self.addCleanup(child.kill)
|
||||||
|
shown, start, modes = b"", 0, []
|
||||||
|
for prompt, act in steps:
|
||||||
|
while prompt not in shown[start:] and (chunk := self._chunk(leader)):
|
||||||
|
shown += chunk
|
||||||
|
if prompt not in shown[start:]:
|
||||||
|
break
|
||||||
|
start = shown.index(prompt, start) + len(prompt)
|
||||||
|
modes.append(self._iutf8(leader))
|
||||||
|
if isinstance(act, bytes):
|
||||||
|
os.write(leader, act)
|
||||||
|
else:
|
||||||
|
time.sleep(0.3)
|
||||||
|
child.send_signal(act)
|
||||||
|
while chunk := self._chunk(leader):
|
||||||
|
shown += chunk
|
||||||
|
return child.wait(timeout=30), shown.decode(errors="replace"), [*modes, self._iutf8(leader)]
|
||||||
|
|
||||||
def _open(self) -> list[str]:
|
def _open(self) -> list[str]:
|
||||||
return [row["task"] for row in json.loads("\n".join(self._out("list", "--json")))]
|
return [row["task"] for row in json.loads("\n".join(self._out("list", "--json")))]
|
||||||
|
|
||||||
@@ -344,6 +401,46 @@ class TestOperator(unittest.TestCase):
|
|||||||
self.assertEqual(self._open(), ["operator/drop"])
|
self.assertEqual(self._open(), ["operator/drop"])
|
||||||
self.assertFalse((self.root / "news.jsonl").exists())
|
self.assertFalse((self.root / "news.jsonl").exists())
|
||||||
|
|
||||||
|
def test_a_cancel_reason_typed_on_a_terminal_keeps_an_erased_umlaut_whole_and_replaces_a_bad_byte(self):
|
||||||
|
self._add("drop", self.base, "true", "r", "manual")
|
||||||
|
self._add("bad", self.base, "true", "r", "manual")
|
||||||
|
code, shown, modes = self._terminal(["run"], [
|
||||||
|
(b"Run it?", b"c\n"),
|
||||||
|
(b"Why cancel?", "ich hä\x7fätte gerne einen befehl bei dem ich seh was passiert\n".encode()),
|
||||||
|
(b"Run it?", b"c\n"),
|
||||||
|
(b"Why cancel?", b"kaputt \xff " + "👀\n".encode()),
|
||||||
|
])
|
||||||
|
self.assertEqual(code, 0, shown)
|
||||||
|
self.assertNotIn("Traceback", shown)
|
||||||
|
self.assertEqual(modes, [True, True, True, True, False])
|
||||||
|
register = self._register()
|
||||||
|
self.assertEqual(register["operator/drop"]["cancelled"], "ich hätte gerne einen befehl bei dem ich seh was passiert")
|
||||||
|
self.assertEqual(register["operator/bad"]["cancelled"], "kaputt � 👀")
|
||||||
|
|
||||||
|
def test_a_granted_cancel_typed_on_a_terminal_keeps_an_erased_umlaut_whole_and_restores_the_mode(self):
|
||||||
|
self._add("paint", self.base, "true", "r", "manual")
|
||||||
|
code, shown, modes = self._terminal(
|
||||||
|
["granted", "operator/paint", "--dir", str(self.base), "--check", "manual", "--", "/bin/sh", "-ic", "true"],
|
||||||
|
[(b"Run it?", b"c\n"), (b"Why cancel?", "ich hä\x7fätte gerne einen befehl\n".encode())],
|
||||||
|
)
|
||||||
|
self.assertEqual((code, modes), (75, [True, True, False]), shown)
|
||||||
|
self.assertEqual(self._register()["operator/paint"]["cancelled"], "ich hätte gerne einen befehl")
|
||||||
|
|
||||||
|
def test_an_interrupt_at_a_prompt_restores_the_terminal_mode(self):
|
||||||
|
self._add("drop", self.base, "true", "r", "manual")
|
||||||
|
code, shown, modes = self._terminal(["run"], [(b"Run it?", b"c\n"), (b"Why cancel?", signal.SIGINT)])
|
||||||
|
self.assertNotEqual(code, 0, shown)
|
||||||
|
self.assertIn("KeyboardInterrupt", shown)
|
||||||
|
self.assertEqual(modes, [True, True, False], shown)
|
||||||
|
self.assertEqual(self._open(), ["operator/drop"])
|
||||||
|
|
||||||
|
def test_run_with_stdin_closed_and_nothing_open_says_so(self):
|
||||||
|
result = subprocess.run(
|
||||||
|
["/bin/sh", "-c", 'exec "$0" "$1" run <&-', sys.executable, str(OPERATOR)],
|
||||||
|
capture_output=True, text=True, env={**self.env, "SHELL": "/bin/sh"}, cwd=self.tmp.name, check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual((result.returncode, result.stdout), (0, "operator: nothing to run\n"), result.stderr)
|
||||||
|
|
||||||
def test_cancel_escapes_the_post_and_works_on_a_granted_row(self):
|
def test_cancel_escapes_the_post_and_works_on_a_granted_row(self):
|
||||||
work = self.base / "work"
|
work = self.base / "work"
|
||||||
work.mkdir()
|
work.mkdir()
|
||||||
|
|||||||
Reference in new issue
Block a user