From 002e515edf7f7339a83dfc0875990bbf1ab9522c Mon Sep 17 00:00:00 2001 From: Hunter Jozwiak Date: Tue, 3 Nov 2026 10:35:00 -0500 Subject: [PATCH] Wait quietly for an absent braille display Helper retries BrlAPI open in-process until BRLTTY answers, supervisor waits with the parked helper instead of kill/respawn cycles, exponential backoff and warn-once for real helper exits, bounded give-up so permanent failures still surface. --- internal/app/braille_helper_test.go | 91 +++++++++++++++++++++++++++++ internal/app/braille_session.go | 77 +++++++++++++++++++----- internal/app/braille_worker.go | 1 + internal/braille/client.go | 44 +++++++++++++- internal/braille/client_test.go | 81 +++++++++++++++++++++++++ 5 files changed, 279 insertions(+), 15 deletions(-) diff --git a/internal/app/braille_helper_test.go b/internal/app/braille_helper_test.go index 2b0fbc7..c3b4741 100644 --- a/internal/app/braille_helper_test.go +++ b/internal/app/braille_helper_test.go @@ -23,6 +23,11 @@ func TestMain(m *testing.M) { fmt.Fprintln(os.Stderr, "braille test startup failure") os.Exit(1) } + if os.Getenv("CTHULHU_BRAILLE_TEST_HANG") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" { + // Simulate BRLTTY stalling the connection attempt (e.g. Bluetooth + // reconnect with the display switched off): never become ready. + select {} + } if os.Getenv("CTHULHU_BRAILLE_TEST_CLIENT") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" { fmt.Println("ready") decoder, encoder := json.NewDecoder(os.Stdin), json.NewEncoder(os.Stdout) @@ -55,6 +60,92 @@ func TestBrailleWorkerReportsHelperStartupFailure(t *testing.T) { } } +// A display that is switched off (or BRLTTY that never answers) makes the +// helper fail every connection attempt. Retries must back off and the +// failure is announced once, not re-warned on every cycle forever. +func TestBrailleWorkerAbsentDisplayBacksOffAndWarnsOnce(t *testing.T) { + if got := nextBrailleBackoff(0); got != time.Second { + t.Fatalf("initial backoff = %v, want 1s", got) + } + if got := nextBrailleBackoff(time.Second); got != 2*time.Second { + t.Fatalf("second backoff = %v, want 2s", got) + } + if got := nextBrailleBackoff(20 * time.Second); got != 30*time.Second { + t.Fatalf("capped backoff = %v, want 30s", got) + } + t.Setenv("CTHULHU_BRAILLE_TEST_FAIL", "1") + path, err := os.Executable() + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + worker := &brailleWorker{path: path, logger: logging.New(&output, logging.Options{Level: slog.LevelDebug}), wake: make(chan struct{}, 1)} + if err := worker.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(3500 * time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := worker.Stop(ctx); err != nil { + t.Fatal(err) + } + if warnings := strings.Count(output.String(), `"message":"braille helper disconnected"`); warnings != 1 { + t.Fatalf("disconnect warnings = %d, want exactly 1\n%s", warnings, output.String()) + } + if worker.Running() { + t.Fatal("worker reports running without a display") + } +} + +// When BRLTTY stalls the connection attempt (display off, Bluetooth +// reconnect in progress), the helper blocks inside Open until BRLTTY +// answers. The session must wait with it instead of killing and respawning; +// cancellation still ends it promptly. +func TestBrailleWorkerSessionWaitsForLateDisplay(t *testing.T) { + t.Setenv("CTHULHU_BRAILLE_TEST_HANG", "1") + path, err := os.Executable() + if err != nil { + t.Fatal(err) + } + worker := &brailleWorker{path: path, logger: slog.New(slog.DiscardHandler), wake: make(chan struct{}, 1)} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- worker.session(ctx) }() + select { + case err := <-done: + t.Fatalf("session gave up on a stalled connection attempt: %v", err) + case <-time.After(6 * time.Second): + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("cancelled session error = %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("cancelled session did not end") + } + if worker.Running() { + t.Fatal("worker reports running while waiting for a display") + } +} + +// A permanently wedged helper (never ready, never exits) is recycled at +// the give-up bound so braille can recover if BRLTTY restarts. +func TestBrailleWorkerSessionRecyclesWedgedHelper(t *testing.T) { + t.Setenv("CTHULHU_BRAILLE_TEST_HANG", "1") + path, err := os.Executable() + if err != nil { + t.Fatal(err) + } + worker := &brailleWorker{path: path, logger: slog.New(slog.DiscardHandler), wake: make(chan struct{}, 1), readyGiveUp: 300 * time.Millisecond} + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := worker.session(ctx); err == nil || !strings.Contains(err.Error(), "connection timed out") { + t.Fatalf("wedged helper error = %v", err) + } +} + func TestBrailleWorkerHelperProtocolAndRestart(t *testing.T) { t.Setenv("CTHULHU_BRAILLE_TEST_CLIENT", "1") path, err := os.Executable() diff --git a/internal/app/braille_session.go b/internal/app/braille_session.go index f43dee1..0f836d4 100644 --- a/internal/app/braille_session.go +++ b/internal/app/braille_session.go @@ -17,16 +17,55 @@ import ( "git.stormux.org/storm/cthulhu/internal/braille" ) +// A helper connection attempt blocks inside BRLTTY until BRLTTY answers, +// which is exactly when a display becomes available: BRLTTY stalls BrlAPI +// while it looks for a switched-off (e.g. Bluetooth) display, and there is +// no cheaper presence query. The session therefore waits with the single +// helper instead of killing and respawning it. brailleReadyGiveUp only +// recycles a permanently wedged helper (e.g. BRLTTY died mid-handshake). +const ( + brailleReadyGiveUp = 5 * time.Minute + brailleReadyNotice = 10 * time.Second +) + +const brailleMaxBackoff = 30 * time.Second + +// nextBrailleBackoff grows the delay between helper connection attempts so +// an absent display (switched off, BRLTTY not answering) is retried on a +// slow cadence instead of respawning the helper every few seconds forever. +func nextBrailleBackoff(current time.Duration) time.Duration { + if current <= 0 { + return time.Second + } + return min(current*2, brailleMaxBackoff) +} + func (worker *brailleWorker) run(ctx context.Context, done chan struct{}) { defer close(done) + backoff := time.Duration(0) + lastError := "" for ctx.Err() == nil { - if err := worker.session(ctx); err != nil && ctx.Err() == nil { - worker.logger.Warn("braille helper disconnected", "error", err) + started := time.Now() + err := worker.session(ctx) + if err != nil && ctx.Err() == nil { + if err.Error() != lastError { + worker.logger.Warn("braille helper disconnected", "error", err) + lastError = err.Error() + } else { + worker.logger.Debug("braille helper still unavailable", "error", err) + } } + // A session that stayed up a while was a real connection; do not + // keep growing the delay after it ends. + if time.Since(started) > 10*time.Second { + backoff = 0 + lastError = "" + } + backoff = nextBrailleBackoff(backoff) select { case <-ctx.Done(): return - case <-time.After(time.Second): + case <-time.After(backoff): } } } @@ -102,17 +141,29 @@ func (worker *brailleWorker) session(ctx context.Context) (result error) { } readErrors <- err }() - timer := time.NewTimer(2 * time.Second) - defer timer.Stop() - select { - case err := <-ready: - if err != nil { - return err + giveUp := worker.readyGiveUp + if giveUp <= 0 { + giveUp = brailleReadyGiveUp + } + giveUpTimer := time.NewTimer(giveUp) + defer giveUpTimer.Stop() + notice := time.NewTimer(brailleReadyNotice) + defer notice.Stop() +readyWait: + for { + select { + case err := <-ready: + if err != nil { + return err + } + break readyWait + case <-notice.C: + worker.logger.Info("waiting for braille display") + case <-giveUpTimer.C: + return errors.New("braille helper connection timed out") + case <-ctx.Done(): + return nil } - case <-timer.C: - return errors.New("braille helper connection timed out") - case <-ctx.Done(): - return nil } worker.logger.Info("braille helper connected") worker.mu.Lock() diff --git a/internal/app/braille_worker.go b/internal/app/braille_worker.go index 5dd67f6..ad0762a 100644 --- a/internal/app/braille_worker.go +++ b/internal/app/braille_worker.go @@ -31,6 +31,7 @@ type brailleWorker struct { connected bool logger *slog.Logger path string + readyGiveUp time.Duration // Wedged-helper recycle bound; zero uses the default. mu sync.Mutex frame braille.Frame baseFrame braille.Frame diff --git a/internal/braille/client.go b/internal/braille/client.go index b81ad73..6f66248 100644 --- a/internal/braille/client.go +++ b/internal/braille/client.go @@ -11,6 +11,43 @@ import ( "time" ) +// openRetryDelay and openRetryGiveUp bound the in-helper wait for a +// display to become ready. Variables so tests can shrink them. +var ( + openRetryDelay = time.Second + openRetryGiveUp = 4 * time.Minute +) + +// openWithRetry waits for a usable display inside the helper. While a +// display is switched off or still negotiating, BRLTTY stalls, refuses, or +// half-answers connection attempts (query failures, zero geometry); each +// is transient, so retrying here claims the display as soon as BRLTTY is +// ready instead of bouncing through supervisor respawn backoff. The last +// error is returned at the give-up bound so permanent misconfiguration +// (e.g. authorization) still surfaces to the parent's log. +func openWithRetry(ctx context.Context, backend Backend) (Geometry, error) { + deadline := time.NewTimer(openRetryGiveUp) + defer deadline.Stop() + for { + geometry, err := backend.Open() + if err == nil { + if _, cellErr := geometry.Cells(); cellErr == nil { + return geometry, nil + } else { + err = cellErr + } + _ = backend.Close() + } + select { + case <-ctx.Done(): + return Geometry{}, ctx.Err() + case <-deadline.C: + return Geometry{}, err + case <-time.After(openRetryDelay): + } + } +} + // RunClient owns BrlAPI inside the disposable helper process. The parent can // terminate this process if a native call blocks or BRLTTY disconnects. func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready io.Writer) (result error) { @@ -18,11 +55,14 @@ func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready // produced an unexpected EOF with the live BRLTTY connection. runtime.LockOSThread() defer runtime.UnlockOSThread() - geometry, err := backend.Open() - defer func() { result = errors.Join(result, backend.Close()) }() + geometry, err := openWithRetry(ctx, backend) if err != nil { + if ctx.Err() != nil { + return nil + } return err } + defer func() { result = errors.Join(result, backend.Close()) }() cells, err := geometry.Cells() if err != nil { return err diff --git a/internal/braille/client_test.go b/internal/braille/client_test.go index 7e22925..3d6a045 100644 --- a/internal/braille/client_test.go +++ b/internal/braille/client_test.go @@ -3,6 +3,7 @@ package braille import ( "bytes" "context" + "errors" "io" "strings" "testing" @@ -37,6 +38,86 @@ func TestRunClientShowsLiveFrameAndReleasesDisplay(t *testing.T) { } } +type flakyOpenBackend struct { + fakeBackend + failures int + cancel context.CancelFunc +} + +func (backend *flakyOpenBackend) Open() (Geometry, error) { + if backend.failures > 0 { + backend.failures-- + backend.opens++ + return Geometry{}, errors.New("display not ready") + } + return backend.fakeBackend.Open() +} + +func (backend *flakyOpenBackend) Write(text string, cursor int) error { + if err := backend.fakeBackend.Write(text, cursor); err != nil { + return err + } + backend.cancel() + return nil +} + +// BRLTTY refuses or half-answers connection attempts while a display is +// still negotiating (observed live: query failures, then zero geometry, +// then success). The helper must keep trying in-process so the display is +// claimed as soon as BRLTTY is ready, without a supervisor respawn cycle. +func TestRunClientRetriesOpenUntilDisplayReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 2, cancel: cancel} + input := strings.NewReader("{\"text\":\"Hello\",\"cursor\":-1}\n") + var ready bytes.Buffer + if err := RunClient(ctx, backend, io.NopCloser(input), &ready); err != nil { + t.Fatal(err) + } + if ready.String() != "ready\n" || backend.opens != 3 || backend.closes != 1 || len(backend.writes) != 1 { + t.Fatalf("ready=%q backend=%+v", ready.String(), backend) + } +} + +// Cancellation while the display has not appeared yet ends the wait +// cleanly and promptly. +func TestRunClientOpenWaitHonorsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 1 << 30, cancel: cancel} + var ready bytes.Buffer + done := make(chan error, 1) + go func() { done <- RunClient(ctx, backend, io.NopCloser(strings.NewReader("")), &ready) }() + time.Sleep(100 * time.Millisecond) + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("cancelled open wait error = %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("open wait ignored cancellation") + } + if ready.Len() != 0 { + t.Fatalf("ready written without a display: %q", ready.String()) + } +} + +// A display that never becomes ready surfaces the last open error at the +// give-up bound instead of waiting silently forever. +func TestRunClientOpenWaitGivesUpWithLastError(t *testing.T) { + delay, giveUp := openRetryDelay, openRetryGiveUp + openRetryDelay, openRetryGiveUp = 10*time.Millisecond, 55*time.Millisecond + defer func() { openRetryDelay, openRetryGiveUp = delay, giveUp }() + backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 1 << 30} + err := RunClient(context.Background(), backend, io.NopCloser(strings.NewReader("")), &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "display not ready") { + t.Fatalf("give-up error = %v", err) + } + if backend.opens < 2 { + t.Fatalf("open attempts = %d, want retries", backend.opens) + } +} + func TestRunClientPlacesCursorOnUnicodeCaret(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel()