Wait quietly for an absent braille display
Helper retries BrlAPI open in-process until BRLTTY answers, supervisor waits with the parked helper instead of kill/respawn cycles, exponential backoff and warn-once for real helper exits, bounded give-up so permanent failures still surface.
This commit is contained in:
1 parent
6c10ed0517
commit
002e515edf
5 files changed
+279
-15
No files matched your search
@@ -23,6 +23,11 @@ func TestMain(m *testing.M) {
|
|||||||
fmt.Fprintln(os.Stderr, "braille test startup failure")
|
fmt.Fprintln(os.Stderr, "braille test startup failure")
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
if os.Getenv("CTHULHU_BRAILLE_TEST_HANG") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" {
|
||||||
|
// Simulate BRLTTY stalling the connection attempt (e.g. Bluetooth
|
||||||
|
// reconnect with the display switched off): never become ready.
|
||||||
|
select {}
|
||||||
|
}
|
||||||
if os.Getenv("CTHULHU_BRAILLE_TEST_CLIENT") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" {
|
if os.Getenv("CTHULHU_BRAILLE_TEST_CLIENT") == "1" && len(os.Args) == 2 && os.Args[1] == "--client" {
|
||||||
fmt.Println("ready")
|
fmt.Println("ready")
|
||||||
decoder, encoder := json.NewDecoder(os.Stdin), json.NewEncoder(os.Stdout)
|
decoder, encoder := json.NewDecoder(os.Stdin), json.NewEncoder(os.Stdout)
|
||||||
@@ -55,6 +60,92 @@ func TestBrailleWorkerReportsHelperStartupFailure(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A display that is switched off (or BRLTTY that never answers) makes the
|
||||||
|
// helper fail every connection attempt. Retries must back off and the
|
||||||
|
// failure is announced once, not re-warned on every cycle forever.
|
||||||
|
func TestBrailleWorkerAbsentDisplayBacksOffAndWarnsOnce(t *testing.T) {
|
||||||
|
if got := nextBrailleBackoff(0); got != time.Second {
|
||||||
|
t.Fatalf("initial backoff = %v, want 1s", got)
|
||||||
|
}
|
||||||
|
if got := nextBrailleBackoff(time.Second); got != 2*time.Second {
|
||||||
|
t.Fatalf("second backoff = %v, want 2s", got)
|
||||||
|
}
|
||||||
|
if got := nextBrailleBackoff(20 * time.Second); got != 30*time.Second {
|
||||||
|
t.Fatalf("capped backoff = %v, want 30s", got)
|
||||||
|
}
|
||||||
|
t.Setenv("CTHULHU_BRAILLE_TEST_FAIL", "1")
|
||||||
|
path, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var output bytes.Buffer
|
||||||
|
worker := &brailleWorker{path: path, logger: logging.New(&output, logging.Options{Level: slog.LevelDebug}), wake: make(chan struct{}, 1)}
|
||||||
|
if err := worker.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(3500 * time.Millisecond)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := worker.Stop(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if warnings := strings.Count(output.String(), `"message":"braille helper disconnected"`); warnings != 1 {
|
||||||
|
t.Fatalf("disconnect warnings = %d, want exactly 1\n%s", warnings, output.String())
|
||||||
|
}
|
||||||
|
if worker.Running() {
|
||||||
|
t.Fatal("worker reports running without a display")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// When BRLTTY stalls the connection attempt (display off, Bluetooth
|
||||||
|
// reconnect in progress), the helper blocks inside Open until BRLTTY
|
||||||
|
// answers. The session must wait with it instead of killing and respawning;
|
||||||
|
// cancellation still ends it promptly.
|
||||||
|
func TestBrailleWorkerSessionWaitsForLateDisplay(t *testing.T) {
|
||||||
|
t.Setenv("CTHULHU_BRAILLE_TEST_HANG", "1")
|
||||||
|
path, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
worker := &brailleWorker{path: path, logger: slog.New(slog.DiscardHandler), wake: make(chan struct{}, 1)}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- worker.session(ctx) }()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("session gave up on a stalled connection attempt: %v", err)
|
||||||
|
case <-time.After(6 * time.Second):
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("cancelled session error = %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("cancelled session did not end")
|
||||||
|
}
|
||||||
|
if worker.Running() {
|
||||||
|
t.Fatal("worker reports running while waiting for a display")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A permanently wedged helper (never ready, never exits) is recycled at
|
||||||
|
// the give-up bound so braille can recover if BRLTTY restarts.
|
||||||
|
func TestBrailleWorkerSessionRecyclesWedgedHelper(t *testing.T) {
|
||||||
|
t.Setenv("CTHULHU_BRAILLE_TEST_HANG", "1")
|
||||||
|
path, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
worker := &brailleWorker{path: path, logger: slog.New(slog.DiscardHandler), wake: make(chan struct{}, 1), readyGiveUp: 300 * time.Millisecond}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := worker.session(ctx); err == nil || !strings.Contains(err.Error(), "connection timed out") {
|
||||||
|
t.Fatalf("wedged helper error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBrailleWorkerHelperProtocolAndRestart(t *testing.T) {
|
func TestBrailleWorkerHelperProtocolAndRestart(t *testing.T) {
|
||||||
t.Setenv("CTHULHU_BRAILLE_TEST_CLIENT", "1")
|
t.Setenv("CTHULHU_BRAILLE_TEST_CLIENT", "1")
|
||||||
path, err := os.Executable()
|
path, err := os.Executable()
|
||||||
|
|||||||
@@ -17,16 +17,55 @@ import (
|
|||||||
"git.stormux.org/storm/cthulhu/internal/braille"
|
"git.stormux.org/storm/cthulhu/internal/braille"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// A helper connection attempt blocks inside BRLTTY until BRLTTY answers,
|
||||||
|
// which is exactly when a display becomes available: BRLTTY stalls BrlAPI
|
||||||
|
// while it looks for a switched-off (e.g. Bluetooth) display, and there is
|
||||||
|
// no cheaper presence query. The session therefore waits with the single
|
||||||
|
// helper instead of killing and respawning it. brailleReadyGiveUp only
|
||||||
|
// recycles a permanently wedged helper (e.g. BRLTTY died mid-handshake).
|
||||||
|
const (
|
||||||
|
brailleReadyGiveUp = 5 * time.Minute
|
||||||
|
brailleReadyNotice = 10 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
const brailleMaxBackoff = 30 * time.Second
|
||||||
|
|
||||||
|
// nextBrailleBackoff grows the delay between helper connection attempts so
|
||||||
|
// an absent display (switched off, BRLTTY not answering) is retried on a
|
||||||
|
// slow cadence instead of respawning the helper every few seconds forever.
|
||||||
|
func nextBrailleBackoff(current time.Duration) time.Duration {
|
||||||
|
if current <= 0 {
|
||||||
|
return time.Second
|
||||||
|
}
|
||||||
|
return min(current*2, brailleMaxBackoff)
|
||||||
|
}
|
||||||
|
|
||||||
func (worker *brailleWorker) run(ctx context.Context, done chan struct{}) {
|
func (worker *brailleWorker) run(ctx context.Context, done chan struct{}) {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
|
backoff := time.Duration(0)
|
||||||
|
lastError := ""
|
||||||
for ctx.Err() == nil {
|
for ctx.Err() == nil {
|
||||||
if err := worker.session(ctx); err != nil && ctx.Err() == nil {
|
started := time.Now()
|
||||||
worker.logger.Warn("braille helper disconnected", "error", err)
|
err := worker.session(ctx)
|
||||||
|
if err != nil && ctx.Err() == nil {
|
||||||
|
if err.Error() != lastError {
|
||||||
|
worker.logger.Warn("braille helper disconnected", "error", err)
|
||||||
|
lastError = err.Error()
|
||||||
|
} else {
|
||||||
|
worker.logger.Debug("braille helper still unavailable", "error", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
// A session that stayed up a while was a real connection; do not
|
||||||
|
// keep growing the delay after it ends.
|
||||||
|
if time.Since(started) > 10*time.Second {
|
||||||
|
backoff = 0
|
||||||
|
lastError = ""
|
||||||
|
}
|
||||||
|
backoff = nextBrailleBackoff(backoff)
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case <-time.After(time.Second):
|
case <-time.After(backoff):
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -102,17 +141,29 @@ func (worker *brailleWorker) session(ctx context.Context) (result error) {
|
|||||||
}
|
}
|
||||||
readErrors <- err
|
readErrors <- err
|
||||||
}()
|
}()
|
||||||
timer := time.NewTimer(2 * time.Second)
|
giveUp := worker.readyGiveUp
|
||||||
defer timer.Stop()
|
if giveUp <= 0 {
|
||||||
select {
|
giveUp = brailleReadyGiveUp
|
||||||
case err := <-ready:
|
}
|
||||||
if err != nil {
|
giveUpTimer := time.NewTimer(giveUp)
|
||||||
return err
|
defer giveUpTimer.Stop()
|
||||||
|
notice := time.NewTimer(brailleReadyNotice)
|
||||||
|
defer notice.Stop()
|
||||||
|
readyWait:
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case err := <-ready:
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
break readyWait
|
||||||
|
case <-notice.C:
|
||||||
|
worker.logger.Info("waiting for braille display")
|
||||||
|
case <-giveUpTimer.C:
|
||||||
|
return errors.New("braille helper connection timed out")
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
case <-timer.C:
|
|
||||||
return errors.New("braille helper connection timed out")
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
worker.logger.Info("braille helper connected")
|
worker.logger.Info("braille helper connected")
|
||||||
worker.mu.Lock()
|
worker.mu.Lock()
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ type brailleWorker struct {
|
|||||||
connected bool
|
connected bool
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
path string
|
path string
|
||||||
|
readyGiveUp time.Duration // Wedged-helper recycle bound; zero uses the default.
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
frame braille.Frame
|
frame braille.Frame
|
||||||
baseFrame braille.Frame
|
baseFrame braille.Frame
|
||||||
|
|||||||
@@ -11,6 +11,43 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// openRetryDelay and openRetryGiveUp bound the in-helper wait for a
|
||||||
|
// display to become ready. Variables so tests can shrink them.
|
||||||
|
var (
|
||||||
|
openRetryDelay = time.Second
|
||||||
|
openRetryGiveUp = 4 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// openWithRetry waits for a usable display inside the helper. While a
|
||||||
|
// display is switched off or still negotiating, BRLTTY stalls, refuses, or
|
||||||
|
// half-answers connection attempts (query failures, zero geometry); each
|
||||||
|
// is transient, so retrying here claims the display as soon as BRLTTY is
|
||||||
|
// ready instead of bouncing through supervisor respawn backoff. The last
|
||||||
|
// error is returned at the give-up bound so permanent misconfiguration
|
||||||
|
// (e.g. authorization) still surfaces to the parent's log.
|
||||||
|
func openWithRetry(ctx context.Context, backend Backend) (Geometry, error) {
|
||||||
|
deadline := time.NewTimer(openRetryGiveUp)
|
||||||
|
defer deadline.Stop()
|
||||||
|
for {
|
||||||
|
geometry, err := backend.Open()
|
||||||
|
if err == nil {
|
||||||
|
if _, cellErr := geometry.Cells(); cellErr == nil {
|
||||||
|
return geometry, nil
|
||||||
|
} else {
|
||||||
|
err = cellErr
|
||||||
|
}
|
||||||
|
_ = backend.Close()
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Geometry{}, ctx.Err()
|
||||||
|
case <-deadline.C:
|
||||||
|
return Geometry{}, err
|
||||||
|
case <-time.After(openRetryDelay):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// RunClient owns BrlAPI inside the disposable helper process. The parent can
|
// RunClient owns BrlAPI inside the disposable helper process. The parent can
|
||||||
// terminate this process if a native call blocks or BRLTTY disconnects.
|
// terminate this process if a native call blocks or BRLTTY disconnects.
|
||||||
func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready io.Writer) (result error) {
|
func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready io.Writer) (result error) {
|
||||||
@@ -18,11 +55,14 @@ func RunClient(ctx context.Context, backend Backend, input io.ReadCloser, ready
|
|||||||
// produced an unexpected EOF with the live BRLTTY connection.
|
// produced an unexpected EOF with the live BRLTTY connection.
|
||||||
runtime.LockOSThread()
|
runtime.LockOSThread()
|
||||||
defer runtime.UnlockOSThread()
|
defer runtime.UnlockOSThread()
|
||||||
geometry, err := backend.Open()
|
geometry, err := openWithRetry(ctx, backend)
|
||||||
defer func() { result = errors.Join(result, backend.Close()) }()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer func() { result = errors.Join(result, backend.Close()) }()
|
||||||
cells, err := geometry.Cells()
|
cells, err := geometry.Cells()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package braille
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -37,6 +38,86 @@ func TestRunClientShowsLiveFrameAndReleasesDisplay(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type flakyOpenBackend struct {
|
||||||
|
fakeBackend
|
||||||
|
failures int
|
||||||
|
cancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
func (backend *flakyOpenBackend) Open() (Geometry, error) {
|
||||||
|
if backend.failures > 0 {
|
||||||
|
backend.failures--
|
||||||
|
backend.opens++
|
||||||
|
return Geometry{}, errors.New("display not ready")
|
||||||
|
}
|
||||||
|
return backend.fakeBackend.Open()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (backend *flakyOpenBackend) Write(text string, cursor int) error {
|
||||||
|
if err := backend.fakeBackend.Write(text, cursor); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
backend.cancel()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BRLTTY refuses or half-answers connection attempts while a display is
|
||||||
|
// still negotiating (observed live: query failures, then zero geometry,
|
||||||
|
// then success). The helper must keep trying in-process so the display is
|
||||||
|
// claimed as soon as BRLTTY is ready, without a supervisor respawn cycle.
|
||||||
|
func TestRunClientRetriesOpenUntilDisplayReady(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 2, cancel: cancel}
|
||||||
|
input := strings.NewReader("{\"text\":\"Hello\",\"cursor\":-1}\n")
|
||||||
|
var ready bytes.Buffer
|
||||||
|
if err := RunClient(ctx, backend, io.NopCloser(input), &ready); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ready.String() != "ready\n" || backend.opens != 3 || backend.closes != 1 || len(backend.writes) != 1 {
|
||||||
|
t.Fatalf("ready=%q backend=%+v", ready.String(), backend)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cancellation while the display has not appeared yet ends the wait
|
||||||
|
// cleanly and promptly.
|
||||||
|
func TestRunClientOpenWaitHonorsCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 1 << 30, cancel: cancel}
|
||||||
|
var ready bytes.Buffer
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- RunClient(ctx, backend, io.NopCloser(strings.NewReader("")), &ready) }()
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("cancelled open wait error = %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("open wait ignored cancellation")
|
||||||
|
}
|
||||||
|
if ready.Len() != 0 {
|
||||||
|
t.Fatalf("ready written without a display: %q", ready.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A display that never becomes ready surfaces the last open error at the
|
||||||
|
// give-up bound instead of waiting silently forever.
|
||||||
|
func TestRunClientOpenWaitGivesUpWithLastError(t *testing.T) {
|
||||||
|
delay, giveUp := openRetryDelay, openRetryGiveUp
|
||||||
|
openRetryDelay, openRetryGiveUp = 10*time.Millisecond, 55*time.Millisecond
|
||||||
|
defer func() { openRetryDelay, openRetryGiveUp = delay, giveUp }()
|
||||||
|
backend := &flakyOpenBackend{fakeBackend: fakeBackend{geometry: Geometry{20, 1}}, failures: 1 << 30}
|
||||||
|
err := RunClient(context.Background(), backend, io.NopCloser(strings.NewReader("")), &bytes.Buffer{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "display not ready") {
|
||||||
|
t.Fatalf("give-up error = %v", err)
|
||||||
|
}
|
||||||
|
if backend.opens < 2 {
|
||||||
|
t.Fatalf("open attempts = %d, want retries", backend.opens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunClientPlacesCursorOnUnicodeCaret(t *testing.T) {
|
func TestRunClientPlacesCursorOnUnicodeCaret(t *testing.T) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|||||||
Reference in new issue
Block a user