From ee1c32f049e19ba6155ec85032bdaeb2a3f9ada1 Mon Sep 17 00:00:00 2001 From: Angus Kola Date: Thu, 5 Nov 2026 23:21:22 -0500 Subject: [PATCH] Verify quarantined Remote mailbox owner crash recovery --- docs/testing/remote-input-helper.md | 28 ++++ .../tentacle_remote_keyboard_native_test.go | 49 ++++++ tests/native_remote_keys.py | 9 +- tests/remote_keyboard_owner_crash.py | 157 ++++++++++++++++++ 4 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 tests/remote_keyboard_owner_crash.py diff --git a/docs/testing/remote-input-helper.md b/docs/testing/remote-input-helper.md index e49820a..cfe0178 100644 --- a/docs/testing/remote-input-helper.md +++ b/docs/testing/remote-input-helper.md @@ -209,6 +209,34 @@ the private mailbox-to-helper path on a responsive synthetic X server, not physical collision safety, production lock/handoff routing or a hard recovery bound. No source Remote command or ordinary reader startup calls this worker. +### Mailbox-owner process failure + +The native harness now also compiles the application race-test executable and +runs the real SDK/host/mailbox worker and Supervisor inside a disposable process. +The source-built helper is launched by that Supervisor in its own process group; +the outer Python fixture never writes helper operations or heartbeats. A separate +X connection verifies both Shift and Control are held before failure. + +Two additional scenarios send SIGKILL or SIGSTOP to the entire owner's process +group. SIGKILL closes the actual Supervisor's lifetime/control writers; SIGSTOP +leaves those writers open but stops its heartbeat owner. In both cases the +independent helper releases both modifiers without orderly reader cleanup. +The stopped process resumes only after release, then checks that the mailbox +session is disabled and the retained Supervisor refuses replacement after the +helper's unconfirmed exit. No automatic input resumption is attempted. + +The outer fixture uses Linux child-subreaper support to adopt and reap orphaned +helper/SDK processes after SIGKILL, without depending on PID 1 or a service +manager. All processes, runtime files and X connections belong to the private +24-bit Xvfb/D-Bus session. Failure cleanup signals only its owned processes. + +The first successful run observed release in 0.01 seconds after SIGKILL and +2.99 seconds after SIGSTOP. These are measurements on a responsive synthetic +server, not recovery deadlines. This exercises the actual quarantined reader +worker's process failure, rather than the earlier synthetic writer alone; it +does not launch or kill an installed/active reader or establish production +reader startup, physical collisions, blocked-server or helper-SIGKILL recovery. + ### Earlier helper and target evidence Run `python3 -B tests/native_remote_keys.py` from the reader checkout. It builds diff --git a/internal/app/tentacle_remote_keyboard_native_test.go b/internal/app/tentacle_remote_keyboard_native_test.go index b511dfb..53d9bd6 100644 --- a/internal/app/tentacle_remote_keyboard_native_test.go +++ b/internal/app/tentacle_remote_keyboard_native_test.go @@ -6,6 +6,7 @@ package app import ( "context" + "errors" "os" "os/exec" "testing" @@ -14,6 +15,54 @@ import ( "git.stormux.org/storm/cthulhu/internal/remoteinput" ) +// The private Python supervisor kills/stops this entire reader-side process, +// leaving its independently grouped helper alive to retire native keys. This +// uses the actual SDK, host, mailbox worker and Supervisor, not a fake writer. +func TestNativeRemoteKeyboardOwnerProcess(t *testing.T) { + ready := os.Getenv("CTHULHU_REMOTE_OWNER_READY") + if os.Getenv("CTHULHU_NATIVE_REMOTE_KEYS") != "1" || ready == "" { + t.Skip("requires private native_remote_keys.py crash supervisor") + } + f := keyboard_owner_fixture(t, func(f *keyboardOwnerFixture) { + var err error + f.supervisor, err = remoteinput.NewSupervisor(os.Getenv("CTHULHU_REMOTE_INPUT_HELPER")) + if err != nil { + t.Fatal(err) + } + }) + lease := f.open(t, "keyboard-inject") + f.key(t, lease, 1, "Shift_L", true) + f.key(t, lease, 2, "Control_R", true) + f.wait(t, func() bool { return keyboard_owner_native_state(t) == "110" }) + if err := os.WriteFile(ready, []byte("ready\n"), 0600); err != nil { + t.Fatal(err) + } + // After SIGSTOP/watchdog expiry the parent resumes us and requests checks. + // SIGKILL never reaches this path or the fixture's orderly cleanup. + var command [1]byte + if n, err := os.Stdin.Read(command[:]); err != nil || n != 1 || command[0] != 'c' { + t.Fatal("crash supervisor command missing", err) + } + select { + case err := <-f.result: + f.result <- err // Preserve the fixture cleanup's join signal. + if !errors.Is(err, remoteinput.ErrUncertain) { + t.Fatal("watchdog helper exit did not retain cleanup uncertainty", err) + } + case <-time.After(6 * time.Second): + t.Fatal("resumed owner did not retire its failed helper") + } + if f.session.OutputContext().Err() == nil { + t.Fatal("resumed owner retained an accepting mailbox") + } + if _, err := f.supervisor.Start(t.Context(), lease.Epoch+1); !errors.Is(err, remoteinput.ErrUnavailable) { + t.Fatal("resumed owner replaced an unconfirmed helper", err) + } + if state := keyboard_owner_native_state(t); state != "000" { + t.Fatal("resumed owner restored retired key state", state) + } +} + func keyboard_owner_native_state(t *testing.T) string { t.Helper() // Query from a separate connection. No key injection or keymap changes diff --git a/tests/native_remote_keys.py b/tests/native_remote_keys.py index 11616f7..21aad7b 100644 --- a/tests/native_remote_keys.py +++ b/tests/native_remote_keys.py @@ -20,11 +20,15 @@ def main(): root = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory(prefix="cthulhu-remote-key-helper-") as temporary: helper = str(Path(temporary) / "cthulhu-input-helper") + owner_test = str(Path(temporary) / "app.test") environment = dict(os.environ, CTHULHU_NATIVE_REMOTE_KEYS="1", CTHULHU_REMOTE_INPUT_HELPER=helper, + CTHULHU_REMOTE_OWNER_TEST=owner_test, GOCACHE="/tmp/cthulhu-cache", GOPROXY="off") subprocess.run(["go", "build", "-race", "-o", helper, "./cmd/cthulhu-input-helper"], cwd=root, env=environment, check=True) + subprocess.run(["go", "test", "-c", "-race", "-o", owner_test, "./internal/app"], + cwd=root, env=environment, check=True) command = ["xvfb-run", "-a", "-s", "-screen 0 1024x768x24", "dbus-run-session", "--", "go", "test", "-race", "./internal/platform/xorg", "-count=1", "-timeout=90s", "-run", "^TestNativeRemoteKey(Target|Helper)", "-v"] @@ -32,7 +36,10 @@ def main(): owner_command = ["xvfb-run", "-a", "-s", "-screen 0 1024x768x24", "dbus-run-session", "--", "go", "test", "-race", "./internal/app", "-count=1", "-timeout=90s", "-run", "^TestNativeRemoteKeyboardOwner$", "-v"] - return run_private_session(owner_command, env=environment, cwd=root, check=True).returncode + run_private_session(owner_command, env=environment, cwd=root, check=True) + crash_command = ["xvfb-run", "-a", "-s", "-screen 0 1024x768x24", "dbus-run-session", "--", + sys.executable, "-B", "tests/remote_keyboard_owner_crash.py"] + return run_private_session(crash_command, env=environment, cwd=root, check=True).returncode if __name__ == "__main__": diff --git a/tests/remote_keyboard_owner_crash.py b/tests/remote_keyboard_owner_crash.py new file mode 100644 index 0000000..1335b08 --- /dev/null +++ b/tests/remote_keyboard_owner_crash.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Storm Dragon +# Authors: Storm Dragon, Angus Kola +# SPDX-License-Identifier: GPL-3.0-or-later +"""Crash the real quarantined mailbox owner, only inside native_remote_keys.py.""" +import ctypes as C +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import time + + +def key_state(): + x = C.CDLL("libX11.so.6") + x.XOpenDisplay.argtypes, x.XOpenDisplay.restype = [C.c_char_p], C.c_void_p + x.XStringToKeysym.argtypes, x.XStringToKeysym.restype = [C.c_char_p], C.c_ulong + x.XKeysymToKeycode.argtypes, x.XKeysymToKeycode.restype = [C.c_void_p, C.c_ulong], C.c_ubyte + x.XQueryKeymap.argtypes = [C.c_void_p, C.c_void_p] + x.XCloseDisplay.argtypes = [C.c_void_p] + display = x.XOpenDisplay(None) + if not display: + raise RuntimeError("Private X key-state connection unavailable") + try: + keys = (C.c_ubyte * 32)() + if not x.XQueryKeymap(display, keys): + raise RuntimeError("Private X key-state query failed") + result = [] + for name in (b"Shift_L", b"Control_R", b"a"): + code = x.XKeysymToKeycode(display, x.XStringToKeysym(name)) + if not code: + raise RuntimeError("Private test key is unavailable") + result.append(bool(keys[code // 8] & (1 << (code % 8)))) + return tuple(result) + finally: + x.XCloseDisplay(display) + + +def wait_for(predicate, message, timeout=7): + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() >= deadline: + raise RuntimeError(message) + time.sleep(0.01) + + +def children(pid): + result = set() + try: + tasks = list(Path(f"/proc/{pid}/task").iterdir()) + except FileNotFoundError: + return result + for task in tasks: + try: + result.update(int(value) for value in (task / "children").read_text().split()) + except FileNotFoundError: + pass # A thread may exit between the directory and children reads. + return result + + +def scenario(mode): + with tempfile.TemporaryDirectory(prefix="cthulhu-owner-crash-") as temporary: + ready = Path(temporary) / "ready" + environment = dict(os.environ, CTHULHU_REMOTE_OWNER_READY=str(ready), TMPDIR=temporary) + with tempfile.TemporaryFile(mode="w+b") as log: + owner = subprocess.Popen( + [environment["CTHULHU_REMOTE_OWNER_TEST"], + "-test.run=^TestNativeRemoteKeyboardOwnerProcess$", "-test.timeout=30s", "-test.v"], + env=environment, stdin=subprocess.PIPE, stdout=log, stderr=log, + start_new_session=True) + descendants = set() + try: + def is_ready(): + if owner.poll() is not None: + raise RuntimeError("Reader-side owner exited before readiness") + return ready.exists() + + wait_for(is_ready, "Reader-side owner did not hold both modifiers", timeout=12) + if key_state() != (True, True, False): + raise RuntimeError("Readiness did not retain the expected native keys") + descendants = children(owner.pid) + helper_path = Path(environment["CTHULHU_REMOTE_INPUT_HELPER"]).resolve() + helpers = [pid for pid in descendants if Path(f"/proc/{pid}/exe").resolve() == helper_path] + if len(helpers) != 1 or len(descendants) != 2: + raise RuntimeError("Expected exactly the source helper and SDK peer") + helper = helpers[0] + if os.getpgid(helper) == os.getpgid(owner.pid): + raise RuntimeError("Helper shared the crashed reader's process group") + began = time.monotonic() + # Crash/stop the reader's entire group, preserving the helper's + # independent group. No graceful cancellation runs in the owner. + os.killpg(owner.pid, signal.SIGKILL if mode == "kill" else signal.SIGSTOP) + wait_for(lambda: key_state() == (False, False, False), + "Independent helper did not release the crashed owner's keys") + elapsed = time.monotonic() - began + if mode == "stop": + # A stopped owner keeps its lifetime descriptor open: only + # the helper's independent heartbeat watchdog can retire it. + if elapsed < 2: + raise RuntimeError("Stopped-owner recovery preceded watchdog expiry") + os.killpg(owner.pid, signal.SIGCONT) + owner.stdin.write(b"c") + owner.stdin.flush() + if owner.wait(timeout=10) != 0: + raise RuntimeError("Resumed owner did not retain retirement barriers") + else: + if owner.wait(timeout=5) != -signal.SIGKILL: + raise RuntimeError("Reader-side owner was not killed as intended") + + def reaped(): + for pid in list(descendants): + ended, _ = os.waitpid(pid, os.WNOHANG) + if ended: + descendants.remove(pid) + return not descendants + + wait_for(reaped, "Orphaned helper or SDK peer did not exit") + if key_state() != (False, False, False): + raise RuntimeError("Retired owner restored native key state") + print(f"PASS mailbox owner SIG{mode.upper()}: keys released in {elapsed:.2f}s", flush=True) + except BaseException: + log.seek(0) + print(log.read().decode(errors="replace"), flush=True) + raise + finally: + # Only disposable fixture processes are killed here. Production + # retirement never kills a helper to satisfy a timeout. + if owner.poll() is None: + descendants.update(children(owner.pid)) + os.killpg(owner.pid, signal.SIGKILL) + owner.wait(timeout=5) + owner.stdin.close() + # The resumed Go owner may already have reaped these PIDs. + # Signal only descendants currently adopted by this supervisor. + for pid in descendants & children(os.getpid()): + try: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + except (ProcessLookupError, ChildProcessError): + pass + + +def main(): + if os.environ.get("CTHULHU_NATIVE_REMOTE_KEYS") != "1": + raise RuntimeError("Run only through native_remote_keys.py") + # Adopt orphaned helpers/SDK peers after owner SIGKILL, without relying on + # PID 1 or an init system to reap them. This process lives only in the fixture. + libc = C.CDLL(None, use_errno=True) + if libc.prctl(36, 1, 0, 0, 0) != 0: # PR_SET_CHILD_SUBREAPER + raise OSError(C.get_errno(), "Cannot adopt private fixture descendants") + for mode in ("kill", "stop"): + scenario(mode) + + +if __name__ == "__main__": + main()