Wait quietly for an absent braille display
Helper retries BrlAPI open in-process until BRLTTY answers, supervisor waits with the parked helper instead of kill/respawn cycles, exponential backoff and warn-once for real helper exits, bounded give-up so permanent failures still surface.
This commit is contained in:
1 parent
6c10ed0517
commit
002e515edf
5 files changed
+279
-15
No files matched your search
@@ -23,6 +23,11 @@ func TestMain(m *testing.M) {
|
||||
fmt.Fprintln(os.Stderr, "braille test startup failure")
|
||||
os.Exit(1)
|
||||
}
|
||||
if os.Getenv("CTHULHU_BRAILLE_TEST_HANG") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" {
|
||||
// Simulate BRLTTY stalling the connection attempt (e.g. Bluetooth
|
||||
// reconnect with the display switched off): never become ready.
|
||||
select {}
|
||||
}
|
||||
if os.Getenv("CTHULHU_BRAILLE_TEST_CLIENT") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" {
|
||||
fmt.Println("ready")
|
||||
decoder, encoder := json.NewDecoder(os.Stdin), json.NewEncoder(os.Stdout)
|
||||
@@ -55,6 +60,92 @@ func TestBrailleWorkerReportsHelperStartupFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A display that is switched off (or BRLTTY that never answers) makes the
|
||||
// helper fail every connection attempt. Retries must back off and the
|
||||
// failure is announced once, not re-warned on every cycle forever.
|
||||
func TestBrailleWorkerAbsentDisplayBacksOffAndWarnsOnce(t *testing.T) {
|
||||
if got := nextBrailleBackoff(0); got != time.Second {
|
||||
t.Fatalf("initial backoff = %v, want 1s", got)
|
||||
}
|
||||
if got := nextBrailleBackoff(time.Second); got != 2*time.Second {
|
||||
t.Fatalf("second backoff = %v, want 2s", got)
|
||||
}
|
||||
if got := nextBrailleBackoff(20 * time.Second); got != 30*time.Second {
|
||||
t.Fatalf("capped backoff = %v, want 30s", got)
|
||||
}
|
||||
t.Setenv("CTHULHU_BRAILLE_TEST_FAIL", "1")
|
||||
path, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
worker := &brailleWorker{path: path, logger: logging.New(&output, logging.Options{Level: slog.LevelDebug}), wake: make(chan struct{}, 1)}
|
||||
if err := worker.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(3500 * time.Millisecond)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := worker.Stop(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if warnings := strings.Count(output.String(), `"message":"braille helper disconnected"`); warnings != 1 {
|
||||
t.Fatalf("disconnect warnings = %d, want exactly 1\n%s", warnings, output.String())
|
||||
}
|
||||
if worker.Running() {
|
||||
t.Fatal("worker reports running without a display")
|
||||
}
|
||||
}
|
||||
|
||||
// When BRLTTY stalls the connection attempt (display off, Bluetooth
|
||||
// reconnect in progress), the helper blocks inside Open until BRLTTY
|
||||
// answers. The session must wait with it instead of killing and respawning;
|
||||
// cancellation still ends it promptly.
|
||||
func TestBrailleWorkerSessionWaitsForLateDisplay(t *testing.T) {
|
||||
t.Setenv("CTHULHU_BRAILLE_TEST_HANG", "1")
|
||||
path, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
worker := &brailleWorker{path: path, logger: slog.New(slog.DiscardHandler), wake: make(chan struct{}, 1)}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- worker.session(ctx) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("session gave up on a stalled connection attempt: %v", err)
|
||||
case <-time.After(6 * time.Second):
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("cancelled session error = %v", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("cancelled session did not end")
|
||||
}
|
||||
if worker.Running() {
|
||||
t.Fatal("worker reports running while waiting for a display")
|
||||
}
|
||||
}
|
||||
|
||||
// A permanently wedged helper (never ready, never exits) is recycled at
|
||||
// the give-up bound so braille can recover if BRLTTY restarts.
|
||||
func TestBrailleWorkerSessionRecyclesWedgedHelper(t *testing.T) {
|
||||
t.Setenv("CTHULHU_BRAILLE_TEST_HANG", "1")
|
||||
path, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
worker := &brailleWorker{path: path, logger: slog.New(slog.DiscardHandler), wake: make(chan struct{}, 1), readyGiveUp: 300 * time.Millisecond}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := worker.session(ctx); err == nil || !strings.Contains(err.Error(), "connection timed out") {
|
||||
t.Fatalf("wedged helper error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrailleWorkerHelperProtocolAndRestart(t *testing.T) {
|
||||
t.Setenv("CTHULHU_BRAILLE_TEST_CLIENT", "1")
|
||||
path, err := os.Executable()
|
||||
|
||||
@@ -17,16 +17,55 @@ import (
|
||||
"git.stormux.org/storm/cthulhu/internal/braille"
|
||||
)
|
||||
|
||||
// A helper connection attempt blocks inside BRLTTY until BRLTTY answers,
|
||||
// which is exactly when a display becomes available: BRLTTY stalls BrlAPI
|
||||
// while it looks for a switched-off (e.g. Bluetooth) display, and there is
|
||||
// no cheaper presence query. The session therefore waits with the single
|
||||
// helper instead of killing and respawning it. brailleReadyGiveUp only
|
||||
// recycles a permanently wedged helper (e.g. BRLTTY died mid-handshake).
|
||||
const (
|
||||
brailleReadyGiveUp = 5 * time.Minute
|
||||
brailleReadyNotice = 10 * time.Second
|
||||
)
|
||||
|
||||
const brailleMaxBackoff = 30 * time.Second
|
||||
|
||||
// nextBrailleBackoff grows the delay between helper connection attempts so
|
||||
// an absent display (switched off, BRLTTY not answering) is retried on a
|
||||
// slow cadence instead of respawning the helper every few seconds forever.
|
||||
func nextBrailleBackoff(current time.Duration) time.Duration {
|
||||
if current <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
return min(current*2, brailleMaxBackoff)
|
||||
}
|
||||
|
||||
func (worker *brailleWorker) run(ctx context.Context, done chan struct{}) {
|
||||
defer close(done)
|
||||
backoff := time.Duration(0)
|
||||
lastError := ""
|
||||
for ctx.Err() == nil {
|
||||
if err := worker.session(ctx); err != nil && ctx.Err() == nil {
|
||||
worker.logger.Warn("braille helper disconnected", "error", err)
|
||||
started := time.Now()
|
||||
err := worker.session(ctx)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
if err.Error() != lastError {
|
||||
worker.logger.Warn("braille helper disconnected", "error", err)
|
||||
lastError = err.Error()
|
||||
} else {
|
||||
worker.logger.Debug("braille helper still unavailable", "error", err)
|
||||
}
|
||||
}
|
||||
// A session that stayed up a while was a real connection; do not
|
||||
// keep growing the delay after it ends.
|
||||
if time.Since(started) > 10*time.Second {
|
||||
backoff = 0
|
||||
lastError = ""
|
||||
}
|
||||
backoff = nextBrailleBackoff(backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -102,17 +141,29 @@ func (worker *brailleWorker) session(ctx context.Context) (result error) {
|
||||
}
|
||||
readErrors <- err
|
||||
}()
|
||||
timer := time.NewTimer(2 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case err := <-ready:
|
||||
if err != nil {
|
||||
return err
|
||||
giveUp := worker.readyGiveUp
|
||||
if giveUp <= 0 {
|
||||
giveUp = brailleReadyGiveUp
|
||||
}
|
||||
giveUpTimer := time.NewTimer(giveUp)
|
||||
defer giveUpTimer.Stop()
|
||||
notice := time.NewTimer(brailleReadyNotice)
|
||||
defer notice.Stop()
|
||||
readyWait:
|
||||
for {
|
||||
select {
|
||||
case err := <-ready:
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
break readyWait
|
||||
case <-notice.C:
|
||||
worker.logger.Info("waiting for braille display")
|
||||
case <-giveUpTimer.C:
|
||||
return errors.New("braille helper connection timed out")
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
}
|
||||
case <-timer.C:
|
||||
return errors.New("braille helper connection timed out")
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
}
|
||||
worker.logger.Info("braille helper connected")
|
||||
worker.mu.Lock()
|
||||
|
||||
@@ -31,6 +31,7 @@ type brailleWorker struct {
|
||||
connected bool
|
||||
logger *slog.Logger
|
||||
path string
|
||||
readyGiveUp time.Duration // Wedged-helper recycle bound; zero uses the default.
|
||||
mu sync.Mutex
|
||||
frame braille.Frame
|
||||
baseFrame braille.Frame
|
||||
|
||||
@@ -11,6 +11,43 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// openRetryDelay and openRetryGiveUp bound the in-helper wait for a
|
||||
// display to become ready. Variables so tests can shrink them.
|
||||
var (
|
||||
openRetryDelay = time.Second
|
||||
openRetryGiveUp = 4 * time.Minute
|
||||
)
|
||||
|
||||
// openWithRetry waits for a usable display inside the helper. While a
|
||||
// display is switched off or still negotiating, BRLTTY stalls, refuses, or
|
||||
// half-answers connection attempts (query failures, zero geometry); each
|
||||
// is transient, so retrying here claims the display as soon as BRLTTY is
|
||||
// ready instead of bouncing through supervisor respawn backoff. The last
|
||||
// error is returned at the give-up bound so permanent misconfiguration
|
||||
// (e.g. authorization) still surfaces to the parent's log.
|
||||
func openWithRetry(ctx context.Context, backend Backend) (Geometry, error) {
|
||||
deadline := time.NewTimer(openRetryGiveUp)
|
||||
defer deadline.Stop()
|
||||
for {
|
||||
geometry, err := backend.Open()
|
||||
if err == nil {
|
||||
if _, cellErr := geometry.Cells(); cellErr == nil {
|
||||
return geometry, nil
|
||||
} else {
|
||||
err = cellErr
|
||||
}
|
||||
_ = backend.Close()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return Geometry{}, ctx.Err()
|
||||
case <-deadline.C:
|
||||
return Geometry{}, err
|
||||
case <-time.After(openRetryDelay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RunClient owns BrlAPI inside the disposable helper process. The parent can
|
||||
// terminate this process if a native call blocks or BRLTTY disconnects.
|
||||
func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready io.Writer) (result error) {
|
||||
@@ -18,11 +55,14 @@ func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready
|
||||
// produced an unexpected EOF with the live BRLTTY connection.
|
||||
runtime.LockOSThread()
|
||||
defer runtime.UnlockOSThread()
|
||||
geometry, err := backend.Open()
|
||||
defer func() { result = errors.Join(result, backend.Close()) }()
|
||||
geometry, err := openWithRetry(ctx, backend)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer func() { result = errors.Join(result, backend.Close()) }()
|
||||
cells, err := geometry.Cells()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -3,6 +3,7 @@ package braille
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -37,6 +38,86 @@ func TestRunClientShowsLiveFrameAndReleasesDisplay(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type flakyOpenBackend struct {
|
||||
fakeBackend
|
||||
failures int
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (backend *flakyOpenBackend) Open() (Geometry, error) {
|
||||
if backend.failures > 0 {
|
||||
backend.failures--
|
||||
backend.opens++
|
||||
return Geometry{}, errors.New("display not ready")
|
||||
}
|
||||
return backend.fakeBackend.Open()
|
||||
}
|
||||
|
||||
func (backend *flakyOpenBackend) Write(text string, cursor int) error {
|
||||
if err := backend.fakeBackend.Write(text, cursor); err != nil {
|
||||
return err
|
||||
}
|
||||
backend.cancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
// BRLTTY refuses or half-answers connection attempts while a display is
|
||||
// still negotiating (observed live: query failures, then zero geometry,
|
||||
// then success). The helper must keep trying in-process so the display is
|
||||
// claimed as soon as BRLTTY is ready, without a supervisor respawn cycle.
|
||||
func TestRunClientRetriesOpenUntilDisplayReady(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 2, cancel: cancel}
|
||||
input := strings.NewReader("{\"text\":\"Hello\",\"cursor\":-1}\n")
|
||||
var ready bytes.Buffer
|
||||
if err := RunClient(ctx, backend, io.NopCloser(input), &ready); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ready.String() != "ready\n" || backend.opens != 3 || backend.closes != 1 || len(backend.writes) != 1 {
|
||||
t.Fatalf("ready=%q backend=%+v", ready.String(), backend)
|
||||
}
|
||||
}
|
||||
|
||||
// Cancellation while the display has not appeared yet ends the wait
|
||||
// cleanly and promptly.
|
||||
func TestRunClientOpenWaitHonorsCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 1 << 30, cancel: cancel}
|
||||
var ready bytes.Buffer
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- RunClient(ctx, backend, io.NopCloser(strings.NewReader("")), &ready) }()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("cancelled open wait error = %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("open wait ignored cancellation")
|
||||
}
|
||||
if ready.Len() != 0 {
|
||||
t.Fatalf("ready written without a display: %q", ready.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A display that never becomes ready surfaces the last open error at the
|
||||
// give-up bound instead of waiting silently forever.
|
||||
func TestRunClientOpenWaitGivesUpWithLastError(t *testing.T) {
|
||||
delay, giveUp := openRetryDelay, openRetryGiveUp
|
||||
openRetryDelay, openRetryGiveUp = 10*time.Millisecond, 55*time.Millisecond
|
||||
defer func() { openRetryDelay, openRetryGiveUp = delay, giveUp }()
|
||||
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 1 << 30}
|
||||
err := RunClient(context.Background(), backend, io.NopCloser(strings.NewReader("")), &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), "display not ready") {
|
||||
t.Fatalf("give-up error = %v", err)
|
||||
}
|
||||
if backend.opens < 2 {
|
||||
t.Fatalf("open attempts = %d, want retries", backend.opens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunClientPlacesCursorOnUnicodeCaret(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
|
||||
Reference in new issue
Block a user