From 2ed3628d9b04dbbf965a45fd585b33e999197111 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 14 Nov 2026 17:31:05 +0000 Subject: [PATCH] fix(kvb-org-operator): keep UTF-8 input whole at every prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit operator.py run crashed with UnicodeDecodeError when the operator erased an umlaut on a terminal without IUTF8: the erase key removed only the second byte of the character and left a lone 0xc3 for input() to decode strictly. typed() and timed() now set IUTF8 on a terminal stdin while they wait for a line and restore the previous mode afterwards, and run replaces undecodable bytes instead of crashing. 🎯 Fix: 95 % 💥 Break: 2 % 💸 Tokens: 100 % ⏱️ Velocity: 100 % 🛡️ Security: ~100 % 🏆 Score: 93 🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 --- skills/kvb-org-operator/scripts/operator.py | 22 +++++ tests/test_operator.py | 97 +++++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/skills/kvb-org-operator/scripts/operator.py b/skills/kvb-org-operator/scripts/operator.py index fab06a9..00146a1 100755 --- a/skills/kvb-org-operator/scripts/operator.py +++ b/skills/kvb-org-operator/scripts/operator.py @@ -136,6 +136,7 @@ import contextlib import errno import hashlib import importlib.util +import io import json import re import select @@ -521,6 +522,24 @@ def ask(holdable: bool, read: Callable[[str], str | None]) -> str: return answer.strip().lower() +def whole_characters(read: Callable[[str], str | None]) -> Callable[[str], str | None]: + """Returns: + read with IUTF8 set on a terminal stdin while it waits for a line and + the previous terminal mode restored afterwards. + """ + def reading(prompt: str) -> str | None: + if not sys.stdin.isatty(): + return read(prompt) + mode = termios.tcgetattr(sys.stdin.fileno()) + termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, [mode[0] | termios.IUTF8, *mode[1:]]) + try: + return read(prompt) + finally: + termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, mode) + return reading + + +@whole_characters def typed(prompt: str) -> str | None: try: return input(prompt) @@ -528,6 +547,7 @@ def typed(prompt: str) -> str | None: return None +@whole_characters def timed(prompt: str) -> str | None: """Returns: One line read unbuffered from stdin, None at its end or when no line @@ -899,6 +919,8 @@ def cmd_run(rasci: ModuleType, args: argparse.Namespace) -> int: shell = os.environ.get("SHELL") if not shell: fail("SHELL is not set; run executes each command in the operator's own shell") + if isinstance(sys.stdin, io.TextIOWrapper): + sys.stdin.reconfigure(errors="replace") steps = [row for row in open_rows(rasci, None) if runnable(row)] if not steps: print("operator: nothing to run") diff --git a/tests/test_operator.py b/tests/test_operator.py index 1424e9e..10c665b 100644 --- a/tests/test_operator.py +++ b/tests/test_operator.py @@ -5,10 +5,13 @@ from __future__ import annotations import json import os import pty +import select import shutil +import signal import subprocess import sys import tempfile +import termios import time import unittest from pathlib import Path @@ -121,6 +124,60 @@ class TestOperator(unittest.TestCase): chunks.append(chunk) return b"".join(chunks).decode() + @staticmethod + def _chunk(leader: int) -> bytes: + if not select.select([leader], [], [], 30)[0]: + return b"" + try: + return os.read(leader, 65536) + except OSError: + return b"" + + @staticmethod + def _iutf8(leader: int) -> bool: + return bool(termios.tcgetattr(leader)[0] & termios.IUTF8) + + def _terminal(self, args: list[str], steps: list[tuple[bytes, bytes | signal.Signals]]) -> tuple[int, str, list[bool]]: + """Args: + args: the operator.py arguments. + steps: each prompt to wait for and what the operator does at it: + the keys typed, or a signal sent once operator.py waits in + the read. + + Returns: + The exit code of operator.py on a terminal without IUTF8 under a + strict UTF-8 stdin, everything the terminal showed, and whether + IUTF8 was set at each prompt reached and after the exit. + """ + leader, follower = pty.openpty() + self.addCleanup(os.close, leader) + mode = termios.tcgetattr(follower) + termios.tcsetattr(follower, termios.TCSANOW, [mode[0] & ~termios.IUTF8, *mode[1:]]) + try: + child = subprocess.Popen( + [sys.executable, str(OPERATOR), *args], stdin=follower, stdout=follower, stderr=follower, + env={**self.env, "SHELL": "/bin/sh", "PYTHONIOENCODING": "utf-8:strict"}, cwd=self.tmp.name, + ) + finally: + os.close(follower) + self.addCleanup(child.kill) + shown, start, modes = b"", 0, [] + for prompt, act in steps: + while prompt not in shown[start:] and (chunk := self._chunk(leader)): + shown += chunk + if prompt not in shown[start:]: + break + start = shown.index(prompt, start) + len(prompt) + modes.append(self._iutf8(leader)) + if isinstance(act, bytes): + os.write(leader, act) + else: + time.sleep(0.3) + child.send_signal(act) + while chunk := self._chunk(leader): + shown += chunk + return child.wait(timeout=30), shown.decode(errors="replace"), [*modes, self._iutf8(leader)] + def _open(self) -> list[str]: return [row["task"] for row in json.loads("\n".join(self._out("list", "--json")))] @@ -344,6 +401,46 @@ class TestOperator(unittest.TestCase): self.assertEqual(self._open(), ["operator/drop"]) self.assertFalse((self.root / "news.jsonl").exists()) + def test_a_cancel_reason_typed_on_a_terminal_keeps_an_erased_umlaut_whole_and_replaces_a_bad_byte(self): + self._add("drop", self.base, "true", "r", "manual") + self._add("bad", self.base, "true", "r", "manual") + code, shown, modes = self._terminal(["run"], [ + (b"Run it?", b"c\n"), + (b"Why cancel?", "ich hä\x7fätte gerne einen befehl bei dem ich seh was passiert\n".encode()), + (b"Run it?", b"c\n"), + (b"Why cancel?", b"kaputt \xff " + "👀\n".encode()), + ]) + self.assertEqual(code, 0, shown) + self.assertNotIn("Traceback", shown) + self.assertEqual(modes, [True, True, True, True, False]) + register = self._register() + self.assertEqual(register["operator/drop"]["cancelled"], "ich hätte gerne einen befehl bei dem ich seh was passiert") + self.assertEqual(register["operator/bad"]["cancelled"], "kaputt � 👀") + + def test_a_granted_cancel_typed_on_a_terminal_keeps_an_erased_umlaut_whole_and_restores_the_mode(self): + self._add("paint", self.base, "true", "r", "manual") + code, shown, modes = self._terminal( + ["granted", "operator/paint", "--dir", str(self.base), "--check", "manual", "--", "/bin/sh", "-ic", "true"], + [(b"Run it?", b"c\n"), (b"Why cancel?", "ich hä\x7fätte gerne einen befehl\n".encode())], + ) + self.assertEqual((code, modes), (75, [True, True, False]), shown) + self.assertEqual(self._register()["operator/paint"]["cancelled"], "ich hätte gerne einen befehl") + + def test_an_interrupt_at_a_prompt_restores_the_terminal_mode(self): + self._add("drop", self.base, "true", "r", "manual") + code, shown, modes = self._terminal(["run"], [(b"Run it?", b"c\n"), (b"Why cancel?", signal.SIGINT)]) + self.assertNotEqual(code, 0, shown) + self.assertIn("KeyboardInterrupt", shown) + self.assertEqual(modes, [True, True, False], shown) + self.assertEqual(self._open(), ["operator/drop"]) + + def test_run_with_stdin_closed_and_nothing_open_says_so(self): + result = subprocess.run( + ["/bin/sh", "-c", 'exec "$0" "$1" run <&-', sys.executable, str(OPERATOR)], + capture_output=True, text=True, env={**self.env, "SHELL": "/bin/sh"}, cwd=self.tmp.name, check=False, + ) + self.assertEqual((result.returncode, result.stdout), (0, "operator: nothing to run\n"), result.stderr) + def test_cancel_escapes_the_post_and_works_on_a_granted_row(self): work = self.base / "work" work.mkdir()