Attempt to make audio and speech discovery more robust.
This commit is contained in:
Storm Dragon committed
1 parent
a27f274099
commit
dc945ff392
21 files changed
+1314
-60
No files matched your search
@@ -0,0 +1,210 @@
|
||||
// Copyright (c) 2026 Storm Dragon
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package sound
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.stormux.org/storm/fenrir/internal/audio"
|
||||
"git.stormux.org/storm/fenrir/internal/presentation"
|
||||
)
|
||||
|
||||
type soundBackend interface {
|
||||
Play(presentation.Cue) bool
|
||||
Play_progress(float64) bool
|
||||
Play_indentation(int) bool
|
||||
Discard_progress()
|
||||
Clear_progress()
|
||||
Set_volume(float64)
|
||||
Errors() <-chan error
|
||||
Finish(context.Context) error
|
||||
Close()
|
||||
}
|
||||
|
||||
type soundConnection struct{ backend soundBackend }
|
||||
|
||||
var errSoundAttemptStalled = errors.New("sound driver open did not return before its deadline")
|
||||
|
||||
// Player keeps connection work away from cue admission and keyboard handling.
|
||||
// Missing output rejects cues so the caller can present their speech equivalent.
|
||||
type Player struct {
|
||||
connection atomic.Pointer[soundConnection]
|
||||
volume atomic.Uint64
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
errors chan error
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func Open_recovering(ctx context.Context, roots []string, options Options) *Player {
|
||||
return new_player(ctx, options.Volume, func() (soundBackend, error) {
|
||||
service, err := Open(roots, options)
|
||||
if service == nil {
|
||||
return nil, err
|
||||
}
|
||||
if err != nil {
|
||||
slog.Warn("sound unavailable or incomplete", "error", err, "at", time.Now())
|
||||
}
|
||||
return service, nil
|
||||
})
|
||||
}
|
||||
|
||||
func new_player(ctx context.Context, volume float64, open func() (soundBackend, error)) *Player {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
p := &Player{cancel: cancel, done: make(chan struct{}), errors: make(chan error, 1)}
|
||||
p.volume.Store(math.Float64bits(volume))
|
||||
// Even a driver whose connection or stream-start call ignores cancellation
|
||||
// cannot hold startup forever. Never retry an attempt that is still stuck.
|
||||
backend, err := sound_attempt(ctx, open)
|
||||
if err == nil {
|
||||
p.connection.Store(&soundConnection{backend})
|
||||
backend.Set_volume(volume)
|
||||
} else {
|
||||
slog.Warn("sound unavailable or incomplete", "reconnecting", true, "error", err, "at", time.Now())
|
||||
}
|
||||
go p.run(ctx, open, err)
|
||||
return p
|
||||
}
|
||||
|
||||
// A timed-out native/library call is quarantined once. A late result is closed
|
||||
// by its own worker; no further opener goroutines are started for this player.
|
||||
func sound_attempt(ctx context.Context, open func() (soundBackend, error)) (soundBackend, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
type result struct {
|
||||
backend soundBackend
|
||||
err error
|
||||
}
|
||||
results := make(chan result)
|
||||
go func() {
|
||||
backend, err := open()
|
||||
select {
|
||||
case results <- result{backend, err}:
|
||||
case <-ctx.Done():
|
||||
if backend != nil {
|
||||
backend.Close()
|
||||
}
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case result := <-results:
|
||||
return result.backend, result.err
|
||||
case <-ctx.Done():
|
||||
return nil, errors.Join(errSoundAttemptStalled, ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Player) run(ctx context.Context, open func() (soundBackend, error), initialError error) {
|
||||
defer close(p.done)
|
||||
problem := initialError
|
||||
for {
|
||||
connection := p.connection.Load()
|
||||
if connection == nil {
|
||||
if errors.Is(problem, errSoundAttemptStalled) || errors.Is(problem, context.Canceled) {
|
||||
if ctx.Err() == nil {
|
||||
p.errors <- fmt.Errorf("sound initialization timed out: %w", problem)
|
||||
}
|
||||
return
|
||||
}
|
||||
retryContext, cancel := context.WithTimeout(ctx, audio.RecoveryTimeout)
|
||||
var backend soundBackend
|
||||
err := audio.Retry(retryContext, func(ctx context.Context) error {
|
||||
var err error
|
||||
backend, err = sound_attempt(ctx, open)
|
||||
return err
|
||||
}, func(err error) bool {
|
||||
return !errors.Is(err, errSoundAttemptStalled) && !errors.Is(err, context.Canceled)
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
p.errors <- fmt.Errorf("sound recovery failed: %w", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
connection = &soundConnection{backend}
|
||||
p.connection.Store(connection)
|
||||
backend.Set_volume(math.Float64frombits(p.volume.Load()))
|
||||
slog.Info("sound connection restored", "at", time.Now())
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case problem = <-connection.backend.Errors():
|
||||
p.connection.Store(nil)
|
||||
connection.backend.Close()
|
||||
slog.Warn("sound connection lost; reconnecting", "error", problem, "at", time.Now())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Player) Errors() <-chan error { return p.errors }
|
||||
|
||||
func (p *Player) Available() bool { return p.connection.Load() != nil }
|
||||
|
||||
func (p *Player) Play(cue presentation.Cue) bool {
|
||||
if connection := p.connection.Load(); connection != nil {
|
||||
return connection.backend.Play(cue)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Player) Play_progress(value float64) bool {
|
||||
if connection := p.connection.Load(); connection != nil {
|
||||
return connection.backend.Play_progress(value)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Player) Play_indentation(level int) bool {
|
||||
if connection := p.connection.Load(); connection != nil {
|
||||
return connection.backend.Play_indentation(level)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *Player) Discard_progress() {
|
||||
if connection := p.connection.Load(); connection != nil {
|
||||
connection.backend.Discard_progress()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Player) Clear_progress() {
|
||||
if connection := p.connection.Load(); connection != nil {
|
||||
connection.backend.Clear_progress()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Player) Set_volume(volume float64) {
|
||||
p.volume.Store(math.Float64bits(volume))
|
||||
if connection := p.connection.Load(); connection != nil {
|
||||
connection.backend.Set_volume(volume)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Player) Finish(ctx context.Context) error {
|
||||
p.cancel()
|
||||
<-p.done
|
||||
if connection := p.connection.Load(); connection != nil {
|
||||
return connection.backend.Finish(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Player) Close() {
|
||||
p.once.Do(func() {
|
||||
p.cancel()
|
||||
<-p.done
|
||||
if connection := p.connection.Swap(nil); connection != nil {
|
||||
connection.backend.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// Copyright (c) 2026 Storm Dragon
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package sound
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.stormux.org/storm/fenrir/internal/presentation"
|
||||
)
|
||||
|
||||
type fakeBackend struct {
|
||||
errors chan error
|
||||
closed chan struct{}
|
||||
once sync.Once
|
||||
plays atomic.Int32
|
||||
volume atomic.Uint64
|
||||
}
|
||||
|
||||
func fake_backend() *fakeBackend {
|
||||
return &fakeBackend{errors: make(chan error, 1), closed: make(chan struct{})}
|
||||
}
|
||||
func (f *fakeBackend) Play(presentation.Cue) bool { f.plays.Add(1); return true }
|
||||
func (f *fakeBackend) Play_progress(float64) bool { return true }
|
||||
func (f *fakeBackend) Play_indentation(int) bool { return true }
|
||||
func (f *fakeBackend) Discard_progress() {}
|
||||
func (f *fakeBackend) Clear_progress() {}
|
||||
func (f *fakeBackend) Set_volume(volume float64) { f.volume.Store(math.Float64bits(volume)) }
|
||||
func (f *fakeBackend) Errors() <-chan error { return f.errors }
|
||||
func (f *fakeBackend) Finish(context.Context) error { return nil }
|
||||
func (f *fakeBackend) Close() { f.once.Do(func() { close(f.closed) }) }
|
||||
|
||||
func wait_player(t *testing.T, player *Player, backend *fakeBackend) {
|
||||
t.Helper()
|
||||
deadline := time.NewTimer(time.Second)
|
||||
defer deadline.Stop()
|
||||
tick := time.NewTicker(time.Millisecond)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
if connection := player.connection.Load(); connection != nil && connection.backend == backend && math.Float64frombits(backend.volume.Load()) == math.Float64frombits(player.volume.Load()) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline.C:
|
||||
t.Fatal("sound connection did not recover")
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSoundPlayerRecoversStartupAndDisconnectWithoutReplayingCues(t *testing.T) {
|
||||
first, second := fake_backend(), fake_backend()
|
||||
firstGate, secondGate := make(chan struct{}), make(chan struct{})
|
||||
attempts := make(chan int, 8)
|
||||
count := 0
|
||||
player := new_player(context.Background(), 0.7, func() (soundBackend, error) {
|
||||
count++
|
||||
attempts <- count
|
||||
switch count {
|
||||
case 1:
|
||||
// A completed library request timeout is safe to retry. It is
|
||||
// distinct from an opener that is still running after our deadline.
|
||||
return nil, context.DeadlineExceeded
|
||||
case 2:
|
||||
<-firstGate
|
||||
return first, nil
|
||||
default:
|
||||
<-secondGate
|
||||
return second, nil
|
||||
}
|
||||
})
|
||||
defer player.Close()
|
||||
<-attempts
|
||||
<-attempts
|
||||
if player.Play(presentation.Ready) || player.Play_progress(50) || player.Play_indentation(2) {
|
||||
t.Fatal("offline sound suppressed speech fallback")
|
||||
}
|
||||
player.Set_volume(0.4)
|
||||
close(firstGate)
|
||||
wait_player(t, player, first)
|
||||
if !player.Play(presentation.Ready) {
|
||||
t.Fatal("recovered sound rejected cue")
|
||||
}
|
||||
first.errors <- errors.New("server disconnected")
|
||||
<-attempts
|
||||
if player.Play(presentation.Error) {
|
||||
t.Fatal("disconnected sound accepted stale cue")
|
||||
}
|
||||
player.Set_volume(0.2)
|
||||
close(secondGate)
|
||||
wait_player(t, player, second)
|
||||
if second.plays.Load() != 0 || first.plays.Load() != 1 {
|
||||
t.Fatal("recovery replayed old cues", first.plays.Load(), second.plays.Load())
|
||||
}
|
||||
select {
|
||||
case <-first.closed:
|
||||
default:
|
||||
t.Fatal("old sound connection leaked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnresponsiveSoundAttemptIsCancelledAndLateResultClosed(t *testing.T) {
|
||||
gate := make(chan struct{})
|
||||
backend := fake_backend()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
started := time.Now()
|
||||
got, err := sound_attempt(ctx, func() (soundBackend, error) { <-gate; return backend, nil })
|
||||
if got != nil || !errors.Is(err, context.DeadlineExceeded) || time.Since(started) > time.Second {
|
||||
t.Fatal("sound attempt blocked caller", got, err)
|
||||
}
|
||||
close(gate)
|
||||
select {
|
||||
case <-backend.closed:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("late sound result leaked")
|
||||
}
|
||||
attempts := 0
|
||||
player := new_player(context.Background(), 0.7, func() (soundBackend, error) {
|
||||
attempts++
|
||||
return nil, errors.Join(errSoundAttemptStalled, context.DeadlineExceeded)
|
||||
})
|
||||
defer player.Close()
|
||||
select {
|
||||
case <-player.Errors():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("unresponsive attempt was retried")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Fatal("overlapping sound opener calls", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSoundPlayerCloseDoesNotWaitForHungReconnect(t *testing.T) {
|
||||
backend := fake_backend()
|
||||
gate, entered := make(chan struct{}), make(chan struct{})
|
||||
attempts := 0
|
||||
player := new_player(context.Background(), 0.7, func() (soundBackend, error) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
return nil, syscall.ENOENT
|
||||
}
|
||||
close(entered)
|
||||
<-gate
|
||||
return backend, nil
|
||||
})
|
||||
<-entered
|
||||
closed := make(chan struct{})
|
||||
go func() { player.Close(); close(closed) }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("shutdown waited for hung sound server")
|
||||
}
|
||||
close(gate)
|
||||
select {
|
||||
case <-backend.closed:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("late sound connection leaked")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user