Wait quietly for an absent braille display

Helper retries BrlAPI open in-process until BRLTTY answers, supervisor waits with the parked helper instead of kill/respawn cycles, exponential backoff and warn-once for real helper exits, bounded give-up so permanent failures still surface.
This commit is contained in:
hjozwiak committed 2026-11-03 10:35:00 -05:00
1 parent 6c10ed0517
commit 002e515edf
5 files changed
+279 -15

No files matched your search

+91
View File
@@ -23,6 +23,11 @@ func TestMain(m *testing.M) {
fmt.Fprintln(os.Stderr, "braille test startup failure")
os.Exit(1)
}
if os.Getenv("CTHULHU_BRAILLE_TEST_HANG") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" {
// Simulate BRLTTY stalling the connection attempt (e.g. Bluetooth
// reconnect with the display switched off): never become ready.
select {}
}
if os.Getenv("CTHULHU_BRAILLE_TEST_CLIENT") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" {
fmt.Println("ready")
decoder, encoder := json.NewDecoder(os.Stdin), json.NewEncoder(os.Stdout)
@@ -55,6 +60,92 @@ func TestBrailleWorkerReportsHelperStartupFailure(t *testing.T) {
}
}
// A display that is switched off (or BRLTTY that never answers) makes the
// helper fail every connection attempt. Retries must back off and the
// failure is announced once, not re-warned on every cycle forever.
func TestBrailleWorkerAbsentDisplayBacksOffAndWarnsOnce(t *testing.T) {
if got := nextBrailleBackoff(0); got != time.Second {
t.Fatalf("initial backoff = %v, want 1s", got)
}
if got := nextBrailleBackoff(time.Second); got != 2*time.Second {
t.Fatalf("second backoff = %v, want 2s", got)
}
if got := nextBrailleBackoff(20 * time.Second); got != 30*time.Second {
t.Fatalf("capped backoff = %v, want 30s", got)
}
t.Setenv("CTHULHU_BRAILLE_TEST_FAIL", "1")
path, err := os.Executable()
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
worker := &brailleWorker{path: path, logger: logging.New(&output, logging.Options{Level: slog.LevelDebug}), wake: make(chan struct{}, 1)}
if err := worker.Start(); err != nil {
t.Fatal(err)
}
time.Sleep(3500 * time.Millisecond)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
if err := worker.Stop(ctx); err != nil {
t.Fatal(err)
}
if warnings := strings.Count(output.String(), `"message":"braille helper disconnected"`); warnings != 1 {
t.Fatalf("disconnect warnings = %d, want exactly 1\n%s", warnings, output.String())
}
if worker.Running() {
t.Fatal("worker reports running without a display")
}
}
// When BRLTTY stalls the connection attempt (display off, Bluetooth
// reconnect in progress), the helper blocks inside Open until BRLTTY
// answers. The session must wait with it instead of killing and respawning;
// cancellation still ends it promptly.
func TestBrailleWorkerSessionWaitsForLateDisplay(t *testing.T) {
t.Setenv("CTHULHU_BRAILLE_TEST_HANG", "1")
path, err := os.Executable()
if err != nil {
t.Fatal(err)
}
worker := &brailleWorker{path: path, logger: slog.New(slog.DiscardHandler), wake: make(chan struct{}, 1)}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- worker.session(ctx) }()
select {
case err := <-done:
t.Fatalf("session gave up on a stalled connection attempt: %v", err)
case <-time.After(6 * time.Second):
}
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("cancelled session error = %v", err)
}
case <-time.After(3 * time.Second):
t.Fatal("cancelled session did not end")
}
if worker.Running() {
t.Fatal("worker reports running while waiting for a display")
}
}
// A permanently wedged helper (never ready, never exits) is recycled at
// the give-up bound so braille can recover if BRLTTY restarts.
func TestBrailleWorkerSessionRecyclesWedgedHelper(t *testing.T) {
t.Setenv("CTHULHU_BRAILLE_TEST_HANG", "1")
path, err := os.Executable()
if err != nil {
t.Fatal(err)
}
worker := &brailleWorker{path: path, logger: slog.New(slog.DiscardHandler), wake: make(chan struct{}, 1), readyGiveUp: 300 * time.Millisecond}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
if err := worker.session(ctx); err == nil || !strings.Contains(err.Error(), "connection timed out") {
t.Fatalf("wedged helper error = %v", err)
}
}
func TestBrailleWorkerHelperProtocolAndRestart(t *testing.T) {
t.Setenv("CTHULHU_BRAILLE_TEST_CLIENT", "1")
path, err := os.Executable()
+64 -13
View File
@@ -17,16 +17,55 @@ import (
"git.stormux.org/storm/cthulhu/internal/braille"
)
// A helper connection attempt blocks inside BRLTTY until BRLTTY answers,
// which is exactly when a display becomes available: BRLTTY stalls BrlAPI
// while it looks for a switched-off (e.g. Bluetooth) display, and there is
// no cheaper presence query. The session therefore waits with the single
// helper instead of killing and respawning it. brailleReadyGiveUp only
// recycles a permanently wedged helper (e.g. BRLTTY died mid-handshake).
const (
brailleReadyGiveUp = 5 * time.Minute
brailleReadyNotice = 10 * time.Second
)
const brailleMaxBackoff = 30 * time.Second
// nextBrailleBackoff grows the delay between helper connection attempts so
// an absent display (switched off, BRLTTY not answering) is retried on a
// slow cadence instead of respawning the helper every few seconds forever.
func nextBrailleBackoff(current time.Duration) time.Duration {
if current <= 0 {
return time.Second
}
return min(current*2, brailleMaxBackoff)
}
func (worker *brailleWorker) run(ctx context.Context, done chan struct{}) {
defer close(done)
backoff := time.Duration(0)
lastError := ""
for ctx.Err() == nil {
if err := worker.session(ctx); err != nil && ctx.Err() == nil {
worker.logger.Warn("braille helper disconnected", "error", err)
started := time.Now()
err := worker.session(ctx)
if err != nil && ctx.Err() == nil {
if err.Error() != lastError {
worker.logger.Warn("braille helper disconnected", "error", err)
lastError = err.Error()
} else {
worker.logger.Debug("braille helper still unavailable", "error", err)
}
}
// A session that stayed up a while was a real connection; do not
// keep growing the delay after it ends.
if time.Since(started) > 10*time.Second {
backoff = 0
lastError = ""
}
backoff = nextBrailleBackoff(backoff)
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
case <-time.After(backoff):
}
}
}
@@ -102,17 +141,29 @@ func (worker *brailleWorker) session(ctx context.Context) (result error) {
}
readErrors <- err
}()
timer := time.NewTimer(2 * time.Second)
defer timer.Stop()
select {
case err := <-ready:
if err != nil {
return err
giveUp := worker.readyGiveUp
if giveUp <= 0 {
giveUp = brailleReadyGiveUp
}
giveUpTimer := time.NewTimer(giveUp)
defer giveUpTimer.Stop()
notice := time.NewTimer(brailleReadyNotice)
defer notice.Stop()
readyWait:
for {
select {
case err := <-ready:
if err != nil {
return err
}
break readyWait
case <-notice.C:
worker.logger.Info("waiting for braille display")
case <-giveUpTimer.C:
return errors.New("braille helper connection timed out")
case <-ctx.Done():
return nil
}
case <-timer.C:
return errors.New("braille helper connection timed out")
case <-ctx.Done():
return nil
}
worker.logger.Info("braille helper connected")
worker.mu.Lock()
+1
View File
@@ -31,6 +31,7 @@ type brailleWorker struct {
connected bool
logger *slog.Logger
path string
readyGiveUp time.Duration // Wedged-helper recycle bound; zero uses the default.
mu sync.Mutex
frame braille.Frame
baseFrame braille.Frame
+42 -2
View File
@@ -11,6 +11,43 @@ import (
"time"
)
// openRetryDelay and openRetryGiveUp bound the in-helper wait for a
// display to become ready. Variables so tests can shrink them.
var (
openRetryDelay = time.Second
openRetryGiveUp = 4 * time.Minute
)
// openWithRetry waits for a usable display inside the helper. While a
// display is switched off or still negotiating, BRLTTY stalls, refuses, or
// half-answers connection attempts (query failures, zero geometry); each
// is transient, so retrying here claims the display as soon as BRLTTY is
// ready instead of bouncing through supervisor respawn backoff. The last
// error is returned at the give-up bound so permanent misconfiguration
// (e.g. authorization) still surfaces to the parent's log.
func openWithRetry(ctx context.Context, backend Backend) (Geometry, error) {
deadline := time.NewTimer(openRetryGiveUp)
defer deadline.Stop()
for {
geometry, err := backend.Open()
if err == nil {
if _, cellErr := geometry.Cells(); cellErr == nil {
return geometry, nil
} else {
err = cellErr
}
_ = backend.Close()
}
select {
case <-ctx.Done():
return Geometry{}, ctx.Err()
case <-deadline.C:
return Geometry{}, err
case <-time.After(openRetryDelay):
}
}
}
// RunClient owns BrlAPI inside the disposable helper process. The parent can
// terminate this process if a native call blocks or BRLTTY disconnects.
func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready io.Writer) (result error) {
@@ -18,11 +55,14 @@ func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready
// produced an unexpected EOF with the live BRLTTY connection.
runtime.LockOSThread()
defer runtime.UnlockOSThread()
geometry, err := backend.Open()
defer func() { result = errors.Join(result, backend.Close()) }()
geometry, err := openWithRetry(ctx, backend)
if err != nil {
if ctx.Err() != nil {
return nil
}
return err
}
defer func() { result = errors.Join(result, backend.Close()) }()
cells, err := geometry.Cells()
if err != nil {
return err
+81
View File
@@ -3,6 +3,7 @@ package braille
import (
"bytes"
"context"
"errors"
"io"
"strings"
"testing"
@@ -37,6 +38,86 @@ func TestRunClientShowsLiveFrameAndReleasesDisplay(t *testing.T) {
}
}
type flakyOpenBackend struct {
fakeBackend
failures int
cancel context.CancelFunc
}
func (backend *flakyOpenBackend) Open() (Geometry, error) {
if backend.failures > 0 {
backend.failures--
backend.opens++
return Geometry{}, errors.New("display not ready")
}
return backend.fakeBackend.Open()
}
func (backend *flakyOpenBackend) Write(text string, cursor int) error {
if err := backend.fakeBackend.Write(text, cursor); err != nil {
return err
}
backend.cancel()
return nil
}
// BRLTTY refuses or half-answers connection attempts while a display is
// still negotiating (observed live: query failures, then zero geometry,
// then success). The helper must keep trying in-process so the display is
// claimed as soon as BRLTTY is ready, without a supervisor respawn cycle.
func TestRunClientRetriesOpenUntilDisplayReady(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 2, cancel: cancel}
input := strings.NewReader("{\"text\":\"Hello\",\"cursor\":-1}\n")
var ready bytes.Buffer
if err := RunClient(ctx, backend, io.NopCloser(input), &ready); err != nil {
t.Fatal(err)
}
if ready.String() != "ready\n" || backend.opens != 3 || backend.closes != 1 || len(backend.writes) != 1 {
t.Fatalf("ready=%q backend=%+v", ready.String(), backend)
}
}
// Cancellation while the display has not appeared yet ends the wait
// cleanly and promptly.
func TestRunClientOpenWaitHonorsCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 1 << 30, cancel: cancel}
var ready bytes.Buffer
done := make(chan error, 1)
go func() { done <- RunClient(ctx, backend, io.NopCloser(strings.NewReader("")), &ready) }()
time.Sleep(100 * time.Millisecond)
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("cancelled open wait error = %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("open wait ignored cancellation")
}
if ready.Len() != 0 {
t.Fatalf("ready written without a display: %q", ready.String())
}
}
// A display that never becomes ready surfaces the last open error at the
// give-up bound instead of waiting silently forever.
func TestRunClientOpenWaitGivesUpWithLastError(t *testing.T) {
delay, giveUp := openRetryDelay, openRetryGiveUp
openRetryDelay, openRetryGiveUp = 10*time.Millisecond, 55*time.Millisecond
defer func() { openRetryDelay, openRetryGiveUp = delay, giveUp }()
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 1 << 30}
err := RunClient(context.Background(), backend, io.NopCloser(strings.NewReader("")), &bytes.Buffer{})
if err == nil || !strings.Contains(err.Error(), "display not ready") {
t.Fatalf("give-up error = %v", err)
}
if backend.opens < 2 {
t.Fatalf("open attempts = %d, want retries", backend.opens)
}
}
func TestRunClientPlacesCursorOnUnicodeCaret(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()