Join quarantined Remote key mailbox to supervised input cleanup

This commit is contained in:
ak47 committed 2026-11-05 23:12:47 -05:00
1 parent 5ab3efdfa7
commit 06530f0120
6 files changed
+711 -6

No files matched your search

+131
View File
@@ -0,0 +1,131 @@
// Copyright (c) 2026 Storm Dragon
// Authors: Storm Dragon, Angus Kola
// SPDX-License-Identifier: GPL-3.0-or-later
package app
import (
"context"
"errors"
"git.stormux.org/storm/cthulhu/internal/remoteinput"
"git.stormux.org/storm/cthulhu/internal/tentacle"
"git.stormux.org/storm/cthulhu/internal/tentaclewire"
)
// run_remote_keyboard is a quarantined mailbox owner, with no production caller.
// The caller must retain one supervisor across sessions, including its permanent
// uncertainty latch, and provide native focus/security policy without native I/O.
// Policy invalidation must cancel ctx or the lease, including while idle. This
// does not establish physical collision safety or trusted command routing.
// Ending the owner disables its session, leaving no accepting orphan mailbox.
func run_remote_keyboard(ctx context.Context, session *tentacle.Session, supervisor *remoteinput.Supervisor,
allow func(tentacle.RemoteKeyDelivery) bool) (result error) {
if ctx == nil || session == nil || supervisor == nil || allow == nil {
return remoteinput.ErrUnavailable
}
defer session.Disable()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
stopOutput := context.AfterFunc(session.OutputContext(), cancel)
defer stopOutput()
incoming := session.EnableRemoteKeys()
var client *remoteinput.Client
var lease tentaclewire.RemoteLease
var helperCancel context.CancelFunc
var stopOwner func() bool
var leaseDone, helperDone <-chan struct{}
retire := func() error {
if client == nil {
return nil
}
// Revoke only this worker's original epoch, never replacement consent.
session.RevokeRemoteEpoch(lease.Epoch)
helperCancel()
stopOwner()
client.Cancel()
err := client.Wait()
client, helperCancel, stopOwner = nil, nil, nil
leaseDone, helperDone = nil, nil
return err
}
defer func() {
if err := retire(); err != nil {
result = err
}
}()
valid := func(delivery tentacle.RemoteKeyDelivery) bool {
if ctx.Err() != nil || delivery.Context == nil || delivery.Context.Err() != nil {
return false
}
lifetime, err := session.RemoteContext(delivery.Lease, "keyboard-inject")
return err == nil && lifetime.Err() == nil && allow(delivery)
}
for {
if ctx.Err() != nil {
return nil
}
select {
case <-ctx.Done():
return nil
case <-leaseDone:
if err := retire(); err != nil {
return err
}
case <-helperDone:
if err := retire(); err != nil {
return err
}
case delivery := <-incoming:
// Keep the host's in-flight budget until native completion or the
// full retirement barrier, including a cancelled/uncertain Apply.
err := func() error {
defer delivery.Release()
if client != nil && lease != delivery.Lease {
if err := retire(); err != nil {
session.RevokeRemoteEpoch(delivery.Lease.Epoch)
return err
}
}
if !valid(delivery) {
session.RevokeRemoteEpoch(delivery.Lease.Epoch)
if client != nil && lease == delivery.Lease {
return retire()
}
return nil
}
if client == nil {
// Derive from the actual original lease, so cancellation is
// visible synchronously even before AfterFunc callbacks run.
helperCtx, cancelHelper := context.WithCancel(delivery.Context)
stop := context.AfterFunc(ctx, cancelHelper)
opened, err := supervisor.Start(helperCtx, delivery.Lease.Epoch)
if err != nil {
stop()
cancelHelper()
session.RevokeRemoteEpoch(delivery.Lease.Epoch)
if errors.Is(err, remoteinput.ErrUncertain) {
return err
}
return nil // A fresh explicit lease may retry a clean startup failure.
}
client, lease = opened, delivery.Lease
helperCancel, stopOwner = cancelHelper, stop
leaseDone, helperDone = delivery.Context.Done(), client.Done()
}
// Launch/readiness may block. Admission and the first check do
// not authorize an effect after the policy or deadline changes.
if !valid(delivery) {
return retire()
}
if err := client.Apply(delivery.Context, delivery.Key.Name, delivery.Key.Pressed); err != nil {
return retire()
}
return nil
}()
if err != nil {
return err
}
}
}
}
@@ -0,0 +1,89 @@
// Copyright (c) 2026 Storm Dragon
// Authors: Storm Dragon, Angus Kola
// SPDX-License-Identifier: GPL-3.0-or-later
package app
import (
"context"
"os"
"os/exec"
"testing"
"time"
"git.stormux.org/storm/cthulhu/internal/remoteinput"
)
func keyboard_owner_native_state(t *testing.T) string {
t.Helper()
// Query from a separate connection. No key injection or keymap changes
// occur in this probe; the actual source-built helper owns all effects.
const script = `import ctypes as C
x = C.CDLL('libX11.so.6')
x.XOpenDisplay.argtypes, x.XOpenDisplay.restype = [C.c_char_p], C.c_void_p
x.XStringToKeysym.argtypes, x.XStringToKeysym.restype = [C.c_char_p], C.c_ulong
x.XKeysymToKeycode.argtypes, x.XKeysymToKeycode.restype = [C.c_void_p, C.c_ulong], C.c_ubyte
x.XQueryKeymap.argtypes = [C.c_void_p, C.c_void_p]
x.XCloseDisplay.argtypes = [C.c_void_p]
d = x.XOpenDisplay(None)
assert d
try:
keys = (C.c_ubyte * 32)()
assert x.XQueryKeymap(d, keys)
for name in (b'Shift_L', b'Control_R', b'a'):
code = x.XKeysymToKeycode(d, x.XStringToKeysym(name))
assert code
print(int(bool(keys[code // 8] & (1 << (code % 8)))), end='')
finally:
x.XCloseDisplay(d)
`
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
output, err := exec.CommandContext(ctx, "python3", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("private X key-state query failed: %v: %s", err, output)
}
return string(output)
}
func TestNativeRemoteKeyboardOwner(t *testing.T) {
if os.Getenv("CTHULHU_NATIVE_REMOTE_KEYS") != "1" {
t.Skip("requires private native_remote_keys.py session")
}
for _, reason := range []string{"policy", "disconnect", "disable", "owner-stop", "replace"} {
t.Run(reason, func(t *testing.T) {
f := keyboard_owner_fixture(t, func(f *keyboardOwnerFixture) {
var err error
f.supervisor, err = remoteinput.NewSupervisor(os.Getenv("CTHULHU_REMOTE_INPUT_HELPER"))
if err != nil {
t.Fatal(err)
}
})
if state := keyboard_owner_native_state(t); state != "000" {
t.Fatal("private server already holds test keys", state)
}
lease := f.open(t, "keyboard-inject")
f.key(t, lease, 1, "Shift_L", true)
f.key(t, lease, 2, "Control_R", true)
f.wait(t, func() bool { return keyboard_owner_native_state(t) == "110" })
switch reason {
case "policy":
f.policy.SetAvailable(false)
f.policy.SetAvailable(true)
case "disconnect":
keyboard_owner_invoke(t, f.session, "rpc", map[string]any{"Method": "remote.close", "Params": map[string]any{"lease": lease.Lease, "epoch": lease.Epoch, "sequence": 3}})
case "disable":
f.session.Disable()
case "owner-stop":
f.cancel()
case "replace":
fresh := f.open(t, "keyboard-inject")
f.key(t, fresh, 1, "a", true)
// Fresh input cannot overtake old native releases/process exit.
f.wait(t, func() bool { return keyboard_owner_native_state(t) == "001" })
f.session.RevokeRemote()
}
f.wait(t, func() bool { return keyboard_owner_native_state(t) == "000" })
})
}
}
@@ -0,0 +1,435 @@
// Copyright (c) 2026 Storm Dragon
// Authors: Storm Dragon, Angus Kola
// SPDX-License-Identifier: GPL-3.0-or-later
package app
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"git.stormux.org/storm/cthulhu/internal/remoteinput"
"git.stormux.org/storm/cthulhu/internal/tentacle"
"git.stormux.org/storm/cthulhu/internal/tentaclewire"
sdk "git.stormux.org/storm/cthulhu/sdk/go/tentacle"
)
// This peer exercises the actual SDK/host path. Consent and the helper target
// are synthetic; no X connection or desktop input is opened by these tests.
func TestRemoteKeyboardPeer(t *testing.T) {
if os.Args[len(os.Args)-1] != "remote-keyboard-peer" {
return
}
client := sdk.NewSessionClient()
commands := []sdk.Command{
{ID: "open", Title: "Open", Handle: func(ctx context.Context, _ json.RawMessage) (any, error) {
for {
select {
case event := <-client.Events():
if event.Kind != "remote.action" {
continue
}
var lease sdk.RemoteLease
err := client.Call(ctx, "remote.open", sdk.RemoteOpen{Action: event.Action, Epoch: event.Epoch, Role: event.Role, Grants: event.Grants}, &lease)
return lease, err
case <-ctx.Done():
return nil, ctx.Err()
}
}
}},
{ID: "rpc", Title: "RPC", Handle: func(ctx context.Context, params json.RawMessage) (any, error) {
var request struct {
Method string
Params json.RawMessage
}
if err := json.Unmarshal(params, &request); err != nil {
return nil, err
}
var result json.RawMessage
err := client.Call(ctx, request.Method, request.Params, &result)
return result, err
}},
}
if sdk.ServeSession(os.Stdin, os.Stdout, commands, nil, client) != nil {
os.Exit(1)
}
os.Exit(0)
}
type keyboardOwnerTestTarget struct{ epoch uint64 }
func (target keyboardOwnerTestTarget) record(event string) {
file, err := os.OpenFile(os.Getenv("CTHULHU_KEY_OWNER_LOG"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
os.Exit(2)
}
_, err = file.WriteString(strconv.FormatUint(target.epoch, 10) + " " + event + "\n")
file.Close()
if err != nil {
os.Exit(2)
}
}
func keyboard_owner_test_gate(name string) {
if path := os.Getenv(name); path != "" {
for {
if _, err := os.Stat(path); err == nil {
return
}
time.Sleep(2 * time.Millisecond)
}
}
}
func (target keyboardOwnerTestTarget) Apply(name string, pressed bool) error {
target.record(name + " " + strconv.FormatBool(pressed))
keyboard_owner_test_gate("CTHULHU_KEY_OWNER_APPLY_GATE")
return nil
}
func (target keyboardOwnerTestTarget) Close() error {
target.record("cleanup")
if os.Getenv("CTHULHU_KEY_OWNER_FAIL_CLEANUP") != "" {
return remoteinput.ErrCleanup
}
return nil
}
func TestRemoteKeyboardHelper(t *testing.T) {
if len(os.Args) < 3 || os.Args[len(os.Args)-2] != "--epoch" {
return
}
epoch, _ := strconv.ParseUint(os.Args[len(os.Args)-1], 10, 64)
control, err := remoteinput.AdoptPipe(os.Stdin, false)
if err != nil {
os.Exit(2)
}
output, err := remoteinput.AdoptPipe(os.Stdout, true)
if err != nil {
os.Exit(2)
}
lifetime, err := remoteinput.AdoptPipe(os.NewFile(3, "lifetime"), false)
if err != nil {
os.Exit(2)
}
target := keyboardOwnerTestTarget{epoch}
target.record("starting")
keyboard_owner_test_gate("CTHULHU_KEY_OWNER_READY_GATE")
if remoteinput.Run(context.Background(), epoch, control, lifetime, output, target) != nil {
os.Exit(1)
}
os.Exit(0)
}
type keyboardOwnerFixture struct {
session *tentacle.Session
policy *tentacle.RemotePolicy
supervisor *remoteinput.Supervisor
allowed atomic.Bool
cancel context.CancelFunc
result chan error
log string
}
func keyboard_owner_fixture(t *testing.T, configure ...func(*keyboardOwnerFixture)) *keyboardOwnerFixture {
t.Helper()
executable, err := os.Executable()
if err != nil {
t.Fatal(err)
}
directory := t.TempDir()
f := &keyboardOwnerFixture{log: filepath.Join(directory, "effects"), result: make(chan error, 1), policy: tentacle.NewRemotePolicy()}
f.allowed.Store(true)
t.Setenv("CTHULHU_KEY_OWNER_LOG", f.log)
program := filepath.Join(directory, "helper")
// The wrapper only adapts test-binary arguments. The real Supervisor still
// owns launch, private pipes, heartbeats, ACK validation and process reaping.
script := "#!/bin/sh\nexec '" + strings.ReplaceAll(executable, "'", "'\\''") + "' -test.run=^TestRemoteKeyboardHelper$ -- \"$@\"\n"
if err := os.WriteFile(program, []byte(script), 0700); err != nil {
t.Fatal(err)
}
f.supervisor, err = remoteinput.NewSupervisor(program)
if err != nil {
t.Fatal(err)
}
manager, err := tentacle.NewManager(tentacle.DefaultLimits(), nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := manager.Shutdown(ctx); err != nil {
t.Error(err)
}
})
if err := manager.UseRemotePolicy(f.policy); err != nil {
t.Fatal(err)
}
f.session, err = manager.Start(t.Context(), tentacle.Spec{ID: "key-owner", Program: []string{executable, "-test.run=^TestRemoteKeyboardPeer$", "--", "remote-keyboard-peer"}, SessionEvents: true, RemoteGrants: []string{"keyboard-inject", "output-observe"}})
if err != nil {
t.Fatal(err)
}
select {
case <-f.session.Ready():
case <-time.After(5 * time.Second):
t.Fatal("keyboard peer did not start")
}
if err := f.session.Activate(); err != nil {
t.Fatal(err)
}
for _, setup := range configure {
setup(f)
}
ctx, cancel := context.WithCancel(t.Context())
f.cancel = cancel
// Attach synchronously, as a future launcher must do before advertising
// keyboard service availability. This call grants no consent.
f.session.EnableRemoteKeys()
go func() {
f.result <- run_remote_keyboard(ctx, f.session, f.supervisor, func(tentacle.RemoteKeyDelivery) bool { return f.allowed.Load() })
}()
t.Cleanup(func() {
// Unblock only these synthetic targets if a test fails before releasing
// its gate. The production supervisor deliberately never kills a helper.
for _, name := range []string{"CTHULHU_KEY_OWNER_READY_GATE", "CTHULHU_KEY_OWNER_APPLY_GATE"} {
if path := os.Getenv(name); path != "" {
_ = os.WriteFile(path, nil, 0600)
}
}
cancel()
select {
case <-f.result:
case <-time.After(6 * time.Second):
t.Error("keyboard owner did not join cleanup")
}
})
return f
}
func keyboard_owner_invoke(t *testing.T, session *tentacle.Session, command string, params any) json.RawMessage {
t.Helper()
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
data, _ := json.Marshal(params)
pending, err := session.Invoke(ctx, command, data)
if err != nil {
t.Fatal(err)
}
select {
case result := <-pending:
if result.Err != nil {
t.Fatal(result.Err)
}
return result.Value
case <-ctx.Done():
t.Fatal("keyboard peer call timed out")
return nil
}
}
func (f *keyboardOwnerFixture) open(t *testing.T, grant string) tentaclewire.RemoteLease {
t.Helper()
approval, err := f.session.BeginRemoteApproval("controlled", []string{grant})
if err != nil {
t.Fatal(err)
}
if _, err := f.session.AuthorizeRemoteWithApproval(approval, "controlled", []string{grant}); err != nil {
t.Fatal(err)
}
var lease tentaclewire.RemoteLease
if json.Unmarshal(keyboard_owner_invoke(t, f.session, "open", nil), &lease) != nil || lease.Epoch == 0 {
t.Fatal("keyboard peer failed to open lease")
}
return lease
}
func (f *keyboardOwnerFixture) key(t *testing.T, lease tentaclewire.RemoteLease, sequence uint64, name string, pressed bool) {
t.Helper()
request := tentaclewire.RemoteKeyRequest{RemoteOperation: tentaclewire.RemoteOperation{Lease: lease.Lease, Epoch: lease.Epoch, Sequence: sequence}, Key: tentaclewire.RemoteKey{Name: name, Pressed: pressed}}
keyboard_owner_invoke(t, f.session, "rpc", map[string]any{"Method": "remote.key", "Params": request})
}
func (f *keyboardOwnerFixture) effects() string {
data, _ := os.ReadFile(f.log)
return string(data)
}
func (f *keyboardOwnerFixture) wait(t *testing.T, predicate func() bool) {
t.Helper()
deadline := time.NewTimer(5 * time.Second)
defer deadline.Stop()
tick := time.NewTicker(2 * time.Millisecond)
defer tick.Stop()
for !predicate() {
select {
case <-tick.C:
case <-deadline.C:
t.Fatal("keyboard owner transition timed out", f.effects())
}
}
}
func TestRemoteKeyboardOwnerOrdersKeysAndReplacementCleanup(t *testing.T) {
f := keyboard_owner_fixture(t)
first := f.open(t, "keyboard-inject")
for index, pressed := range []bool{true, true, false} {
f.key(t, first, uint64(index+1), "Shift_L", pressed)
}
prefix := strconv.FormatUint(first.Epoch, 10) + " "
f.wait(t, func() bool { return strings.Contains(f.effects(), prefix+"Shift_L false\n") })
second := f.open(t, "keyboard-inject") // Replacement may precede old process exit.
f.key(t, second, 1, "Control_R", true)
secondPrefix := strconv.FormatUint(second.Epoch, 10) + " "
f.wait(t, func() bool { return strings.Contains(f.effects(), secondPrefix+"Control_R true\n") })
expected := prefix + "starting\n" + prefix + "Shift_L true\n" + prefix + "Shift_L true\n" + prefix + "Shift_L false\n" + prefix + "cleanup\n" + secondPrefix + "starting\n" + secondPrefix + "Control_R true\n"
if f.effects() != expected {
t.Fatal("key/replacement ordering changed", f.effects())
}
if _, err := f.session.RemoteContext(second, "keyboard-inject"); err != nil {
t.Fatal("old retirement cancelled replacement consent", err)
}
f.policy.SetAvailable(false)
f.policy.SetAvailable(true)
f.wait(t, func() bool { return strings.Contains(f.effects(), secondPrefix+"cleanup\n") })
}
func TestRemoteKeyboardOwnerRechecksPolicyAfterStartup(t *testing.T) {
gate := filepath.Join(t.TempDir(), "ready")
t.Setenv("CTHULHU_KEY_OWNER_READY_GATE", gate)
f := keyboard_owner_fixture(t)
lease := f.open(t, "keyboard-inject")
f.key(t, lease, 1, "a", true)
f.wait(t, func() bool { return strings.Contains(f.effects(), "starting\n") })
f.allowed.Store(false)
if err := os.WriteFile(gate, nil, 0600); err != nil {
t.Fatal(err)
}
f.wait(t, func() bool { return strings.Contains(f.effects(), "cleanup\n") })
if strings.Contains(f.effects(), "a true") {
t.Fatal("startup admission survived changed native policy")
}
if _, err := f.session.RemoteContext(lease, "keyboard-inject"); err == nil {
t.Fatal("policy denial retained the input lease")
}
f.allowed.Store(true)
fresh := f.open(t, "keyboard-inject")
f.key(t, fresh, 1, "b", true)
f.wait(t, func() bool { return strings.Contains(f.effects(), "b true\n") })
}
func TestRemoteKeyboardOwnerDeniedWorkDoesNotLaunchHelper(t *testing.T) {
f := keyboard_owner_fixture(t)
f.allowed.Store(false)
lease := f.open(t, "keyboard-inject")
f.key(t, lease, 1, "a", true)
f.wait(t, func() bool { _, err := f.session.RemoteContext(lease, "keyboard-inject"); return err != nil })
if f.effects() != "" {
t.Fatal("policy-denied work launched a native helper")
}
}
func TestRemoteKeyboardOwnerCancelledStartupAllowsFreshLease(t *testing.T) {
gate := filepath.Join(t.TempDir(), "ready")
t.Setenv("CTHULHU_KEY_OWNER_READY_GATE", gate)
f := keyboard_owner_fixture(t)
old := f.open(t, "keyboard-inject")
f.key(t, old, 1, "a", true)
f.wait(t, func() bool { return strings.Contains(f.effects(), "starting\n") })
fresh := f.open(t, "keyboard-inject")
f.key(t, fresh, 1, "b", true)
if err := os.WriteFile(gate, nil, 0600); err != nil {
t.Fatal(err)
}
f.wait(t, func() bool { return strings.Contains(f.effects(), "b true\n") })
if strings.Contains(f.effects(), "a true") {
t.Fatal("retired startup delivered its old key")
}
if _, err := f.session.RemoteContext(fresh, "keyboard-inject"); err != nil {
t.Fatal("old startup failure revoked fresh consent", err)
}
}
func TestRemoteKeyboardOwnerIdleLeaseExpiryRetiresHelper(t *testing.T) {
f := keyboard_owner_fixture(t)
lease := f.open(t, "keyboard-inject")
f.key(t, lease, 1, "Control_R", true)
f.wait(t, func() bool { return strings.Contains(f.effects(), "Control_R true\n") })
// The Supervisor keeps its private watchdog alive; those heartbeats must
// never extend the authenticated reader lease's independent deadline.
f.wait(t, func() bool { return strings.Contains(f.effects(), "cleanup\n") })
if _, err := f.session.RemoteContext(lease, "keyboard-inject"); err == nil {
t.Fatal("private helper heartbeats renewed reader consent")
}
}
func TestRemoteKeyboardOwnerShutdownJoinsCleanupAndDisablesMailbox(t *testing.T) {
f := keyboard_owner_fixture(t)
lease := f.open(t, "keyboard-inject")
f.key(t, lease, 1, "Control_R", true)
f.wait(t, func() bool { return strings.Contains(f.effects(), "Control_R true\n") })
f.cancel()
select {
case err := <-f.result:
f.result <- err
if err != nil || !strings.Contains(f.effects(), "cleanup\n") {
t.Fatal("owner shutdown did not confirm cleanup", err)
}
case <-time.After(6 * time.Second):
t.Fatal("owner shutdown did not complete")
}
if f.session.OutputContext().Err() == nil {
t.Fatal("stopped owner retained an accepting session")
}
}
func TestRemoteKeyboardOwnerRevokesBlockedAndQueuedKeys(t *testing.T) {
gate := filepath.Join(t.TempDir(), "apply")
t.Setenv("CTHULHU_KEY_OWNER_APPLY_GATE", gate)
f := keyboard_owner_fixture(t)
lease := f.open(t, "keyboard-inject")
f.key(t, lease, 1, "Shift_L", true)
f.wait(t, func() bool { return strings.Contains(f.effects(), "Shift_L true\n") })
f.key(t, lease, 2, "a", true)
f.key(t, lease, 3, "Shift_L", false)
f.session.RevokeRemote()
if err := os.WriteFile(gate, nil, 0600); err != nil {
t.Fatal(err)
}
f.wait(t, func() bool { return strings.Contains(f.effects(), "cleanup\n") })
if strings.Contains(f.effects(), "a true") || strings.Contains(f.effects(), "Shift_L false") {
t.Fatal("revoked queued keys reached the helper", f.effects())
}
}
func TestRemoteKeyboardOwnerUncertainCleanupStopsSession(t *testing.T) {
t.Setenv("CTHULHU_KEY_OWNER_FAIL_CLEANUP", "1")
f := keyboard_owner_fixture(t)
lease := f.open(t, "keyboard-inject")
f.key(t, lease, 1, "Control_L", true)
f.wait(t, func() bool { return strings.Contains(f.effects(), "Control_L true\n") })
f.session.RevokeRemote()
select {
case err := <-f.result:
f.result <- err // Preserve cleanup's join signal.
if !errors.Is(err, remoteinput.ErrUncertain) {
t.Fatal("failed native cleanup was reported as clean", err)
}
case <-time.After(6 * time.Second):
t.Fatal("failed cleanup did not stop the owner")
}
if state, _ := f.session.Status(); state != tentacle.Stopping && state != tentacle.Stopped {
t.Fatal("orphan mailbox session remained enabled", state)
}
if _, err := f.supervisor.Start(t.Context(), lease.Epoch+1); !errors.Is(err, remoteinput.ErrUnavailable) {
t.Fatal("uncertain helper was replaced", err)
}
}