Verify quarantined Remote mailbox owner crash recovery

docs/testing/remote-input-helper.md

@@ -209,6 +209,34 @@ the private mailbox-to-helper path on a responsive synthetic X server, not
 physical collision safety, production lock/handoff routing or a hard recovery
 bound. No source Remote command or ordinary reader startup calls this worker.
 
+### Mailbox-owner process failure
+
+The native harness now also compiles the application race-test executable and
+runs the real SDK/host/mailbox worker and Supervisor inside a disposable process.
+The source-built helper is launched by that Supervisor in its own process group;
+the outer Python fixture never writes helper operations or heartbeats. A separate
+X connection verifies both Shift and Control are held before failure.
+
+Two additional scenarios send SIGKILL or SIGSTOP to the entire owner's process
+group. SIGKILL closes the actual Supervisor's lifetime/control writers; SIGSTOP
+leaves those writers open but stops its heartbeat owner. In both cases the
+independent helper releases both modifiers without orderly reader cleanup.
+The stopped process resumes only after release, then checks that the mailbox
+session is disabled and the retained Supervisor refuses replacement after the
+helper's unconfirmed exit. No automatic input resumption is attempted.
+
+The outer fixture uses Linux child-subreaper support to adopt and reap orphaned
+helper/SDK processes after SIGKILL, without depending on PID 1 or a service
+manager. All processes, runtime files and X connections belong to the private
+24-bit Xvfb/D-Bus session. Failure cleanup signals only its owned processes.
+
+The first successful run observed release in 0.01 seconds after SIGKILL and
+2.99 seconds after SIGSTOP. These are measurements on a responsive synthetic
+server, not recovery deadlines. This exercises the actual quarantined reader
+worker's process failure, rather than the earlier synthetic writer alone; it
+does not launch or kill an installed/active reader or establish production
+reader startup, physical collisions, blocked-server or helper-SIGKILL recovery.
+
 ### Earlier helper and target evidence
 
 Run `python3 -B tests/native_remote_keys.py` from the reader checkout. It builds

internal/app/tentacle_remote_keyboard_native_test.go

@@ -6,6 +6,7 @@ package app
 
 import (
 	"context"
+	"errors"
 	"os"
 	"os/exec"
 	"testing"
@@ -14,6 +15,54 @@ import (
 	"git.stormux.org/storm/cthulhu/internal/remoteinput"
 )
 
+// The private Python supervisor kills/stops this entire reader-side process,
+// leaving its independently grouped helper alive to retire native keys. This
+// uses the actual SDK, host, mailbox worker and Supervisor, not a fake writer.
+func TestNativeRemoteKeyboardOwnerProcess(t *testing.T) {
+	ready := os.Getenv("CTHULHU_REMOTE_OWNER_READY")
+	if os.Getenv("CTHULHU_NATIVE_REMOTE_KEYS") != "1" || ready == "" {
+		t.Skip("requires private native_remote_keys.py crash supervisor")
+	}
+	f := keyboard_owner_fixture(t, func(f *keyboardOwnerFixture) {
+		var err error
+		f.supervisor, err = remoteinput.NewSupervisor(os.Getenv("CTHULHU_REMOTE_INPUT_HELPER"))
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+	lease := f.open(t, "keyboard-inject")
+	f.key(t, lease, 1, "Shift_L", true)
+	f.key(t, lease, 2, "Control_R", true)
+	f.wait(t, func() bool { return keyboard_owner_native_state(t) == "110" })
+	if err := os.WriteFile(ready, []byte("ready\n"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	// After SIGSTOP/watchdog expiry the parent resumes us and requests checks.
+	// SIGKILL never reaches this path or the fixture's orderly cleanup.
+	var command [1]byte
+	if n, err := os.Stdin.Read(command[:]); err != nil || n != 1 || command[0] != 'c' {
+		t.Fatal("crash supervisor command missing", err)
+	}
+	select {
+	case err := <-f.result:
+		f.result <- err // Preserve the fixture cleanup's join signal.
+		if !errors.Is(err, remoteinput.ErrUncertain) {
+			t.Fatal("watchdog helper exit did not retain cleanup uncertainty", err)
+		}
+	case <-time.After(6 * time.Second):
+		t.Fatal("resumed owner did not retire its failed helper")
+	}
+	if f.session.OutputContext().Err() == nil {
+		t.Fatal("resumed owner retained an accepting mailbox")
+	}
+	if _, err := f.supervisor.Start(t.Context(), lease.Epoch+1); !errors.Is(err, remoteinput.ErrUnavailable) {
+		t.Fatal("resumed owner replaced an unconfirmed helper", err)
+	}
+	if state := keyboard_owner_native_state(t); state != "000" {
+		t.Fatal("resumed owner restored retired key state", state)
+	}
+}
+
 func keyboard_owner_native_state(t *testing.T) string {
 	t.Helper()
 	// Query from a separate connection. No key injection or keymap changes

tests/native_remote_keys.py

@@ -20,11 +20,15 @@ def main():
     root = Path(__file__).resolve().parents[1]
     with tempfile.TemporaryDirectory(prefix="cthulhu-remote-key-helper-") as temporary:
         helper = str(Path(temporary) / "cthulhu-input-helper")
+        owner_test = str(Path(temporary) / "app.test")
         environment = dict(os.environ, CTHULHU_NATIVE_REMOTE_KEYS="1",
                            CTHULHU_REMOTE_INPUT_HELPER=helper,
+                           CTHULHU_REMOTE_OWNER_TEST=owner_test,
                            GOCACHE="/tmp/cthulhu-cache", GOPROXY="off")
         subprocess.run(["go", "build", "-race", "-o", helper, "./cmd/cthulhu-input-helper"],
                        cwd=root, env=environment, check=True)
+        subprocess.run(["go", "test", "-c", "-race", "-o", owner_test, "./internal/app"],
+                       cwd=root, env=environment, check=True)
         command = ["xvfb-run", "-a", "-s", "-screen 0 1024x768x24", "dbus-run-session", "--",
                    "go", "test", "-race", "./internal/platform/xorg", "-count=1", "-timeout=90s",
                    "-run", "^TestNativeRemoteKey(Target|Helper)", "-v"]
@@ -32,7 +36,10 @@ def main():
         owner_command = ["xvfb-run", "-a", "-s", "-screen 0 1024x768x24", "dbus-run-session", "--",
                          "go", "test", "-race", "./internal/app", "-count=1", "-timeout=90s",
                          "-run", "^TestNativeRemoteKeyboardOwner$", "-v"]
-        return run_private_session(owner_command, env=environment, cwd=root, check=True).returncode
+        run_private_session(owner_command, env=environment, cwd=root, check=True)
+        crash_command = ["xvfb-run", "-a", "-s", "-screen 0 1024x768x24", "dbus-run-session", "--",
+                         sys.executable, "-B", "tests/remote_keyboard_owner_crash.py"]
+        return run_private_session(crash_command, env=environment, cwd=root, check=True).returncode
 
 
 if __name__ == "__main__":

tests/remote_keyboard_owner_crash.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026 Storm Dragon
+# Authors: Storm Dragon, Angus Kola
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Crash the real quarantined mailbox owner, only inside native_remote_keys.py."""
+import ctypes as C
+import os
+from pathlib import Path
+import signal
+import subprocess
+import tempfile
+import time
+
+
+def key_state():
+    x = C.CDLL("libX11.so.6")
+    x.XOpenDisplay.argtypes, x.XOpenDisplay.restype = [C.c_char_p], C.c_void_p
+    x.XStringToKeysym.argtypes, x.XStringToKeysym.restype = [C.c_char_p], C.c_ulong
+    x.XKeysymToKeycode.argtypes, x.XKeysymToKeycode.restype = [C.c_void_p, C.c_ulong], C.c_ubyte
+    x.XQueryKeymap.argtypes = [C.c_void_p, C.c_void_p]
+    x.XCloseDisplay.argtypes = [C.c_void_p]
+    display = x.XOpenDisplay(None)
+    if not display:
+        raise RuntimeError("Private X key-state connection unavailable")
+    try:
+        keys = (C.c_ubyte * 32)()
+        if not x.XQueryKeymap(display, keys):
+            raise RuntimeError("Private X key-state query failed")
+        result = []
+        for name in (b"Shift_L", b"Control_R", b"a"):
+            code = x.XKeysymToKeycode(display, x.XStringToKeysym(name))
+            if not code:
+                raise RuntimeError("Private test key is unavailable")
+            result.append(bool(keys[code // 8] & (1 << (code % 8))))
+        return tuple(result)
+    finally:
+        x.XCloseDisplay(display)
+
+
+def wait_for(predicate, message, timeout=7):
+    deadline = time.monotonic() + timeout
+    while not predicate():
+        if time.monotonic() >= deadline:
+            raise RuntimeError(message)
+        time.sleep(0.01)
+
+
+def children(pid):
+    result = set()
+    try:
+        tasks = list(Path(f"/proc/{pid}/task").iterdir())
+    except FileNotFoundError:
+        return result
+    for task in tasks:
+        try:
+            result.update(int(value) for value in (task / "children").read_text().split())
+        except FileNotFoundError:
+            pass  # A thread may exit between the directory and children reads.
+    return result
+
+
+def scenario(mode):
+    with tempfile.TemporaryDirectory(prefix="cthulhu-owner-crash-") as temporary:
+        ready = Path(temporary) / "ready"
+        environment = dict(os.environ, CTHULHU_REMOTE_OWNER_READY=str(ready), TMPDIR=temporary)
+        with tempfile.TemporaryFile(mode="w+b") as log:
+            owner = subprocess.Popen(
+                [environment["CTHULHU_REMOTE_OWNER_TEST"],
+                 "-test.run=^TestNativeRemoteKeyboardOwnerProcess$", "-test.timeout=30s", "-test.v"],
+                env=environment, stdin=subprocess.PIPE, stdout=log, stderr=log,
+                start_new_session=True)
+            descendants = set()
+            try:
+                def is_ready():
+                    if owner.poll() is not None:
+                        raise RuntimeError("Reader-side owner exited before readiness")
+                    return ready.exists()
+
+                wait_for(is_ready, "Reader-side owner did not hold both modifiers", timeout=12)
+                if key_state() != (True, True, False):
+                    raise RuntimeError("Readiness did not retain the expected native keys")
+                descendants = children(owner.pid)
+                helper_path = Path(environment["CTHULHU_REMOTE_INPUT_HELPER"]).resolve()
+                helpers = [pid for pid in descendants if Path(f"/proc/{pid}/exe").resolve() == helper_path]
+                if len(helpers) != 1 or len(descendants) != 2:
+                    raise RuntimeError("Expected exactly the source helper and SDK peer")
+                helper = helpers[0]
+                if os.getpgid(helper) == os.getpgid(owner.pid):
+                    raise RuntimeError("Helper shared the crashed reader's process group")
+                began = time.monotonic()
+                # Crash/stop the reader's entire group, preserving the helper's
+                # independent group. No graceful cancellation runs in the owner.
+                os.killpg(owner.pid, signal.SIGKILL if mode == "kill" else signal.SIGSTOP)
+                wait_for(lambda: key_state() == (False, False, False),
+                         "Independent helper did not release the crashed owner's keys")
+                elapsed = time.monotonic() - began
+                if mode == "stop":
+                    # A stopped owner keeps its lifetime descriptor open: only
+                    # the helper's independent heartbeat watchdog can retire it.
+                    if elapsed < 2:
+                        raise RuntimeError("Stopped-owner recovery preceded watchdog expiry")
+                    os.killpg(owner.pid, signal.SIGCONT)
+                    owner.stdin.write(b"c")
+                    owner.stdin.flush()
+                    if owner.wait(timeout=10) != 0:
+                        raise RuntimeError("Resumed owner did not retain retirement barriers")
+                else:
+                    if owner.wait(timeout=5) != -signal.SIGKILL:
+                        raise RuntimeError("Reader-side owner was not killed as intended")
+
+                    def reaped():
+                        for pid in list(descendants):
+                            ended, _ = os.waitpid(pid, os.WNOHANG)
+                            if ended:
+                                descendants.remove(pid)
+                        return not descendants
+
+                    wait_for(reaped, "Orphaned helper or SDK peer did not exit")
+                if key_state() != (False, False, False):
+                    raise RuntimeError("Retired owner restored native key state")
+                print(f"PASS mailbox owner SIG{mode.upper()}: keys released in {elapsed:.2f}s", flush=True)
+            except BaseException:
+                log.seek(0)
+                print(log.read().decode(errors="replace"), flush=True)
+                raise
+            finally:
+                # Only disposable fixture processes are killed here. Production
+                # retirement never kills a helper to satisfy a timeout.
+                if owner.poll() is None:
+                    descendants.update(children(owner.pid))
+                    os.killpg(owner.pid, signal.SIGKILL)
+                    owner.wait(timeout=5)
+                owner.stdin.close()
+                # The resumed Go owner may already have reaped these PIDs.
+                # Signal only descendants currently adopted by this supervisor.
+                for pid in descendants & children(os.getpid()):
+                    try:
+                        os.kill(pid, signal.SIGKILL)
+                        os.waitpid(pid, 0)
+                    except (ProcessLookupError, ChildProcessError):
+                        pass
+
+
+def main():
+    if os.environ.get("CTHULHU_NATIVE_REMOTE_KEYS") != "1":
+        raise RuntimeError("Run only through native_remote_keys.py")
+    # Adopt orphaned helpers/SDK peers after owner SIGKILL, without relying on
+    # PID 1 or an init system to reap them. This process lives only in the fixture.
+    libc = C.CDLL(None, use_errno=True)
+    if libc.prctl(36, 1, 0, 0, 0) != 0:  # PR_SET_CHILD_SUBREAPER
+        raise OSError(C.get_errno(), "Cannot adopt private fixture descendants")
+    for mode in ("kill", "stop"):
+        scenario(mode)
+
+
+if __name__ == "__main__":
+    main()