Attempt to make audio and speech discovery more robust.

This commit is contained in:
Storm Dragon committed 2026-11-03 13:21:50 -05:00
1 parent a27f274099
commit dc945ff392
21 files changed
+1314 -60

No files matched your search

+12 -2
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"net"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
@@ -22,8 +23,17 @@ type serverAddress struct {
// runtime directory selection for console sessions.
func default_socket_path() string {
if uid := os.Getenv("SUDO_UID"); uid != "" {
if _, err := strconv.ParseUint(uid, 10, 32); err == nil {
return filepath.Join("/run/user", uid, "speech-dispatcher/speechd.sock")
if parsed, err := strconv.ParseUint(uid, 10, 32); err == nil {
runtimeDirectory := os.Getenv("XDG_RUNTIME_DIR")
if !owned_directory(runtimeDirectory, uint32(parsed)) {
runtimeDirectory = filepath.Join("/run/user", uid)
}
if !owned_directory(runtimeDirectory, uint32(parsed)) {
if account, err := user.LookupId(uid); err == nil {
runtimeDirectory = filepath.Join(account.HomeDir, ".cache")
}
}
return filepath.Join(runtimeDirectory, "speech-dispatcher/speechd.sock")
}
}
runtimeDirectory := os.Getenv("XDG_RUNTIME_DIR")
+65 -4
View File
@@ -11,6 +11,8 @@ import (
"net"
"os"
"os/exec"
"os/user"
"path/filepath"
"strconv"
"strings"
"syscall"
@@ -102,10 +104,6 @@ func spawn_arguments(ctx context.Context, endpoint serverAddress) ([]string, err
}
func spawn_server(ctx context.Context, endpoint serverAddress) error {
// Never create a root-owned server in the invoking user's runtime directory.
if uid := os.Getenv("SUDO_UID"); uid != "" && uid != strconv.Itoa(os.Geteuid()) {
return fmt.Errorf("start Speech Dispatcher as the invoking user before running Fenrir with sudo")
}
spawnContext, cancel := context.WithTimeout(ctx, commandTimeout)
defer cancel()
args, err := spawn_arguments(spawnContext, endpoint)
@@ -117,6 +115,24 @@ func spawn_server(ctx context.Context, endpoint serverAddress) error {
command = "speech-dispatcher"
}
process := exec.CommandContext(spawnContext, command, args...)
if uid := os.Getenv("SUDO_UID"); uid != "" && uid != strconv.Itoa(os.Geteuid()) {
// Drop credentials only in the child. Never run a root-owned daemon in
// another user's session or change the reader's device privileges.
if os.Geteuid() != 0 {
return fmt.Errorf("cannot start Speech Dispatcher as the invoking user without root privileges")
}
account, err := user.LookupId(uid)
if err != nil {
return fmt.Errorf("look up invoking user for Speech Dispatcher: %w", err)
}
groups, err := account.GroupIds()
if err != nil {
return fmt.Errorf("look up invoking user's groups: %w", err)
}
if err := configure_sudo_spawn(process, account, groups); err != nil {
return err
}
}
process.WaitDelay = 100 * time.Millisecond
output, err := process.CombinedOutput()
if err != nil {
@@ -124,3 +140,48 @@ func spawn_server(ctx context.Context, endpoint serverAddress) error {
}
return nil
}
func configure_sudo_spawn(process *exec.Cmd, account *user.User, groups []string) error {
uid, uidErr := strconv.ParseUint(account.Uid, 10, 32)
gid, gidErr := strconv.ParseUint(account.Gid, 10, 32)
if uidErr != nil || gidErr != nil || uid == 0 || !filepath.IsAbs(account.HomeDir) {
return fmt.Errorf("invalid invoking user for Speech Dispatcher")
}
credentials := &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}
for _, group := range groups {
id, err := strconv.ParseUint(group, 10, 32)
if err != nil {
return fmt.Errorf("invalid invoking user's group: %w", err)
}
credentials.Groups = append(credentials.Groups, uint32(id))
}
process.SysProcAttr = &syscall.SysProcAttr{Credential: credentials}
process.Dir = account.HomeDir
// Do not pass root's HOME, XDG paths, D-Bus address, or unrelated secrets
// into a user-owned daemon. Preserve explicit audio choices and locale.
process.Env = []string{"HOME=" + account.HomeDir, "USER=" + account.Username,
"LOGNAME=" + account.Username, "PATH=/usr/local/bin:/usr/bin:/bin"}
for _, entry := range os.Environ() {
key, _, _ := strings.Cut(entry, "=")
if key == "LANG" || strings.HasPrefix(key, "LC_") || key == "PULSE_SERVER" || key == "PULSE_COOKIE" {
process.Env = append(process.Env, entry)
}
}
runtimeDir := filepath.Join("/run/user", account.Uid)
if candidate := os.Getenv("XDG_RUNTIME_DIR"); owned_directory(candidate, uint32(uid)) {
runtimeDir = candidate
}
if owned_directory(runtimeDir, uint32(uid)) {
process.Env = append(process.Env, "XDG_RUNTIME_DIR="+runtimeDir)
}
return nil
}
func owned_directory(path string, uid uint32) bool {
info, err := os.Stat(path)
if err != nil || !info.IsDir() {
return false
}
stat, ok := info.Sys().(*syscall.Stat_t)
return ok && stat.Uid == uid
}
+4 -1
View File
@@ -149,7 +149,10 @@ func TestDiscoveryDoesNotSpawn(t *testing.T) {
}
}
func TestSudoDoesNotSpawnAsAnotherUser(t *testing.T) {
func TestUnprivilegedProcessDoesNotImpersonateSudoUser(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("requires an unprivileged caller")
}
t.Setenv("SUDO_UID", strconv.Itoa(os.Geteuid()+1))
err := spawn_server(context.Background(), serverAddress{"unix", "/tmp/custom.sock"})
if err == nil || !strings.Contains(err.Error(), "invoking user") {
+11 -4
View File
@@ -4,6 +4,7 @@
package speech
import (
"context"
"fmt"
"os"
"sort"
@@ -45,7 +46,7 @@ func Discover(options Options) (Catalog, error) {
if err != nil {
return result, err
}
defer w.conn.Close()
defer w.close()
deadline := time.Now().Add(4 * time.Second)
query := func(command string) ([]string, error) {
if time.Now().After(deadline) {
@@ -176,6 +177,12 @@ func Language_voices(voices []Voice, language string) []Voice {
// Prepare checks membership because SSIP SET may accept unknown synthesis
// names without verifying them. The resulting client has not spoken anything.
func Prepare(driver string, options Options) (Transport, error) {
return Prepare_context(context.Background(), driver, options)
}
// Prepare_context bounds discovery and connection setup together.
func Prepare_context(ctx context.Context, driver string, options Options) (Transport, error) {
options.context = ctx
if err := Validate_selection(options.Module, options.Language, options.Voice); err != nil {
return nil, err
}
@@ -185,7 +192,7 @@ func Prepare(driver string, options Options) (Transport, error) {
}
if d.Voices == nil {
if options.Module == "" && options.Language == "" && options.Voice == "" {
return Open_driver(driver, options)
return Open_driver_context(ctx, driver, options)
}
return nil, fmt.Errorf("speech driver does not support voice discovery")
}
@@ -219,7 +226,7 @@ func Prepare(driver string, options Options) (Transport, error) {
if !languageFound || !voiceFound {
return nil, fmt.Errorf("speech language or voice is unavailable for module %q", options.Module)
}
return Open_driver(driver, options)
return Open_driver_context(ctx, driver, options)
}
// A live change queries only its target, so a slow unrelated module cannot
@@ -232,7 +239,7 @@ func discover_voices(options Options) ([]Voice, error) {
if err != nil {
return nil, err
}
defer w.conn.Close()
defer w.close()
if options.Module != "" {
if _, err = w.command("SET SELF OUTPUT_MODULE " + options.Module); err != nil {
return nil, err
+56 -13
View File
@@ -43,8 +43,20 @@ type Rejection struct{ Code string }
func (r *Rejection) Error() string { return "Speech Dispatcher rejected a command: " + r.Code }
type wire struct {
conn net.Conn
reader *bufio.Reader
conn net.Conn
reader *bufio.Reader
stopContext func() bool
}
func (w *wire) detach_context() {
if w.stopContext != nil {
w.stopContext()
}
}
func (w *wire) close() {
w.detach_context()
_ = w.conn.Close()
}
func (w *wire) reply() ([]string, error) {
@@ -102,11 +114,14 @@ func connect_options(options Options) (*wire, error) {
if err != nil {
return nil, err
}
return connect_endpoint(endpoint, options.autospawn, automatic_server_address(options.Socket))
return connect_endpoint(options.context, endpoint, options.autospawn, automatic_server_address(options.Socket))
}
func connect_endpoint(endpoint serverAddress, autospawn, systemFallback bool) (*wire, error) {
ctx, cancel := context.WithTimeout(context.Background(), 3*commandTimeout)
func connect_endpoint(parent context.Context, endpoint serverAddress, autospawn, systemFallback bool) (*wire, error) {
if parent == nil {
parent = context.Background()
}
ctx, cancel := context.WithTimeout(parent, 3*commandTimeout)
defer cancel()
dialer := &net.Dialer{Timeout: commandTimeout}
dial := serverDialer(dialer.DialContext)
@@ -123,7 +138,7 @@ func connect_endpoint(endpoint serverAddress, autospawn, systemFallback bool) (*
if err != nil {
return nil, err
}
return &wire{conn: conn, reader: bufio.NewReaderSize(conn, 4096)}, nil
return &wire{conn: conn, reader: bufio.NewReaderSize(conn, 4096), stopContext: context.AfterFunc(parent, func() { _ = conn.Close() })}, nil
}
type Client struct {
@@ -132,11 +147,13 @@ type Client struct {
sendMu sync.Mutex
controlMu sync.Mutex
closeOnce sync.Once
options Options
}
type Options struct {
// Only Prepare enables startup during its prerequisite selection queries.
autospawn bool
context context.Context
Socket string
Rate int
Pitch *int
@@ -174,15 +191,15 @@ func Open(options Options) (*Client, error) {
if err != nil {
return nil, err
}
data, err := connect_endpoint(endpoint, true, automatic_server_address(options.Socket))
data, err := connect_endpoint(options.context, endpoint, true, automatic_server_address(options.Socket))
if err != nil {
return nil, fmt.Errorf("connect Speech Dispatcher at %s: %w", endpoint.address, err)
}
client := &Client{data: data}
client := &Client{data: data, options: options}
fail := func(err error) (*Client, error) {
data.conn.Close()
data.close()
if client.control != nil {
client.control.conn.Close()
client.control.close()
}
return nil, err
}
@@ -229,13 +246,31 @@ func Open(options Options) (*Client, error) {
if endpoint.network == "unix" {
endpoint.address = data.conn.RemoteAddr().String()
}
client.control, err = connect_endpoint(endpoint, false, false)
client.control, err = connect_endpoint(options.context, endpoint, false, false)
if err != nil {
return fail(err)
}
if _, err := client.control.command(`SET SELF CLIENT_NAME "fenrir:fenrir:control"`); err != nil {
return fail(err)
}
data.detach_context()
client.control.detach_context()
if options.context != nil && options.context.Err() != nil {
client.abort()
return nil, options.context.Err()
}
client.options.context = nil
return client, nil
}
// Reopen reconnects only the selected server and synthesis options.
func (c *Client) Reopen(ctx context.Context) (Transport, error) {
options := c.options
options.context = ctx
client, err := Open(options)
if err != nil {
return nil, err
}
return client, nil
}
@@ -315,8 +350,16 @@ func (c *Client) Cancel() error {
func (c *Client) Close() {
c.closeOnce.Do(func() {
// Closing the data socket releases a blocked sender before cancellation.
c.data.conn.Close()
c.data.close()
c.Cancel()
c.control.conn.Close()
c.control.close()
})
}
// abort releases blocked initialization commands without another SSIP request.
func (c *Client) abort() {
_ = c.data.conn.Close()
if c.control != nil {
_ = c.control.conn.Close()
}
}
+6 -3
View File
@@ -21,7 +21,9 @@ import (
"bufio"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
@@ -299,9 +301,10 @@ func TestLiveParametersUseSpeakingClientAndRejectInvalidValues(t *testing.T) {
func TestDefaultSocketForSudo(t *testing.T) {
t.Setenv("SPEECHD_ADDRESS", "")
t.Setenv("SPEECHD_SOCKET", "")
t.Setenv("SUDO_UID", "1000")
t.Setenv("XDG_RUNTIME_DIR", "/run/user/0")
if got, err := resolve_server_address(""); err != nil || got.address != "/run/user/1000/speech-dispatcher/speechd.sock" {
runtimeDirectory := t.TempDir()
t.Setenv("SUDO_UID", strconv.Itoa(os.Getuid()))
t.Setenv("XDG_RUNTIME_DIR", runtimeDirectory)
if got, err := resolve_server_address(""); err != nil || got.address != filepath.Join(runtimeDirectory, "speech-dispatcher/speechd.sock") {
t.Fatal(got, err)
}
t.Setenv("SPEECHD_ADDRESS", "unix_socket:/tmp/example-speechd.sock")
+16 -3
View File
@@ -17,7 +17,10 @@
package speech
import "fmt"
import (
"context"
"fmt"
)
// Driver binds a stable file/CLI ID to its implementation. Availability must
// be a cheap dependency/hardware check, never connect, speak or start a service.
@@ -83,6 +86,12 @@ func Find_driver(id string) (Driver, error) {
}
func Open_driver(id string, options Options) (Transport, error) {
return Open_driver_context(context.Background(), id, options)
}
// Open_driver_context bounds connection setup without binding the live client to ctx.
func Open_driver_context(ctx context.Context, id string, options Options) (Transport, error) {
options.context = ctx
driver, err := Find_driver(id)
if err != nil {
return nil, err
@@ -93,7 +102,11 @@ func Open_driver(id string, options Options) (Transport, error) {
if err = driver.Availability(); err != nil {
return nil, fmt.Errorf("speech driver %s unavailable: %w", id, err)
}
return driver.Open(options)
client, err := driver.Open(options)
if err != nil {
return nil, err
}
return client, nil
}
// discover_modules uses a separate, bounded connection. It never changes the
@@ -103,7 +116,7 @@ func discover_modules(options Options) ([]string, error) {
if err != nil {
return nil, err
}
defer wire.conn.Close()
defer wire.close()
modules, err := wire.command("LIST OUTPUT_MODULES")
if err != nil {
return nil, err
+288
View File
@@ -0,0 +1,288 @@
// Copyright (c) 2026 Storm Dragon
// SPDX-License-Identifier: GPL-3.0-or-later
package speech
import (
"context"
"errors"
"io"
"net"
"os"
"os/exec"
"os/user"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
"git.stormux.org/storm/fenrir/internal/audio"
)
type disconnectedTransport struct {
*recordingTransport
reopen func(context.Context) (Transport, error)
closed atomic.Bool
}
func (f *disconnectedTransport) Say(string) error { return io.EOF }
func (f *disconnectedTransport) Cancel() error { return io.EOF }
func (f *disconnectedTransport) Close() { f.closed.Store(true) }
func (f *disconnectedTransport) Reopen(ctx context.Context) (Transport, error) {
return f.reopen(ctx)
}
func TestRecoveryKeepsAdmissionResponsiveAndDropsOutageSpeech(t *testing.T) {
entered, release := make(chan struct{}), make(chan struct{})
fresh := new_recording_transport()
broken := &disconnectedTransport{recordingTransport: new_recording_transport(), reopen: func(ctx context.Context) (Transport, error) {
close(entered)
select {
case <-release:
return fresh, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}}
service := New(broken)
defer service.Close()
service.Configure(Parameters{Rate: 40, Pitch: 20, Volume: 80})
service.Say("lost")
<-entered
admitted := make(chan struct{})
go func() {
service.Interrupt()
service.Configure(Parameters{Rate: 70, Pitch: 10, Volume: 90})
service.Say("discarded during outage")
service.Toggle_mute()
service.Say("discarded while muted")
close(admitted)
}()
select {
case <-admitted:
case <-time.After(time.Second):
t.Fatal("audio recovery blocked input admission")
}
close(release)
wait_event(t, fresh.changed, "configure:70:10:90")
service.Toggle_mute()
service.Say_with_pitch("fresh", 60)
wait_event(t, fresh.changed, "sent:fresh")
fresh.mu.Lock()
joined := strings.Join(fresh.events, "|")
fresh.mu.Unlock()
if strings.Contains(joined, "lost") || strings.Contains(joined, "discarded") || !strings.Contains(joined, "pitch:60|say:fresh|sent:fresh|pitch:10") || !broken.closed.Load() {
t.Fatal(joined, broken.closed.Load())
}
select {
case err := <-service.Errors():
t.Fatal("successful recovery was fatal", err)
default:
}
}
func TestRecoveryFailureIsFatalAndShutdownCancelsReopen(t *testing.T) {
for _, failure := range []bool{true, false} {
t.Run(strconv.FormatBool(failure), func(t *testing.T) {
entered := make(chan struct{})
broken := &disconnectedTransport{recordingTransport: new_recording_transport(), reopen: func(ctx context.Context) (Transport, error) {
close(entered)
if failure {
return nil, syscall.EACCES
}
<-ctx.Done()
return nil, ctx.Err()
}}
service := New(broken)
service.Say("trigger recovery")
<-entered
if failure {
select {
case err := <-service.Errors():
if !errors.Is(err, syscall.EACCES) {
t.Fatal(err)
}
case <-time.After(time.Second):
t.Fatal("permanent recovery failure was hidden")
}
}
closed := make(chan struct{})
go func() { service.Close(); close(closed) }()
select {
case <-closed:
case <-time.After(time.Second):
t.Fatal("shutdown waited for unavailable speech")
}
})
}
}
func TestSpeechRecoveryDeadlineReportsFailure(t *testing.T) {
broken := &disconnectedTransport{recordingTransport: new_recording_transport(), reopen: func(ctx context.Context) (Transport, error) {
<-ctx.Done()
return nil, ctx.Err()
}}
service := New(broken)
defer service.Close()
ctx, cancel := context.WithTimeout(service.recoveryContext, 40*time.Millisecond)
defer cancel()
service.recoveryContext = ctx
service.Say("trigger unavailable daemon")
select {
case err := <-service.Errors():
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatal(err)
}
case <-time.After(time.Second):
t.Fatal("exhausted recovery left the reader silently running")
}
}
func TestOpenContextCancelsSilentHandshakeAndDoesNotOwnLiveClient(t *testing.T) {
listener, err := net.Listen("unix", filepath.Join(t.TempDir(), "silent.sock"))
if err != nil {
t.Fatal(err)
}
defer listener.Close()
accepted := make(chan net.Conn, 1)
go func() { conn, _ := listener.Accept(); accepted <- conn }()
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond)
defer cancel()
started := time.Now()
client, err := Open_driver_context(ctx, "speech-dispatcher", Options{Socket: listener.Addr().String()})
if client != nil || err == nil || time.Since(started) > time.Second {
t.Fatal("silent handshake ignored cancellation", client, err)
}
(<-accepted).Close()
server := new_fake_server(t, false)
ctx, cancel = context.WithCancel(context.Background())
client, err = Open_driver_context(ctx, "speech-dispatcher", Options{Socket: server.path})
if err != nil {
t.Fatal(err)
}
defer client.Close()
cancel()
if err := client.Say("live client survives startup context"); err != nil {
t.Fatal(err)
}
}
func TestSudoSpawnDropsChildCredentialsAndFiltersRootEnvironment(t *testing.T) {
t.Setenv("HOME", "/root")
t.Setenv("XDG_RUNTIME_DIR", "/root/runtime")
t.Setenv("DBUS_SESSION_BUS_ADDRESS", "unix:path=/root/session-bus")
t.Setenv("FENRIR_TEST_SECRET", "must-not-reach-child")
t.Setenv("PULSE_SERVER", "unix:/tmp/selected-audio.sock")
command := exec.Command("speech-dispatcher", "--spawn")
account := &user.User{Uid: "2000", Gid: "2001", Username: "Username", HomeDir: "/home/Username"}
if err := configure_sudo_spawn(command, account, []string{"2001", "2002"}); err != nil {
t.Fatal(err)
}
want := &syscall.Credential{Uid: 2000, Gid: 2001, Groups: []uint32{2001, 2002}}
if !reflect.DeepEqual(command.SysProcAttr.Credential, want) || command.Dir != account.HomeDir {
t.Fatal(command.SysProcAttr, command.Dir)
}
environment := strings.Join(command.Env, "\n")
if strings.Contains(environment, "/root") || strings.Contains(environment, "FENRIR_TEST_SECRET") || !strings.Contains(environment, "HOME=/home/Username") || !strings.Contains(environment, "PULSE_SERVER=unix:/tmp/selected-audio.sock") {
t.Fatal(environment)
}
account.Uid = "0"
if err := configure_sudo_spawn(command, account, nil); err == nil {
t.Fatal("invalid invoking user selected root")
}
}
func TestStartupWithoutSudoRetriesUntilPrivateDaemonAppears(t *testing.T) {
t.Setenv("SUDO_UID", "")
t.Setenv("SPEECHD_CMD", "/nonexistent-fixture-speech-dispatcher")
socket := filepath.Join(t.TempDir(), "speech.sock")
t.Setenv("SPEECHD_ADDRESS", "unix_socket:"+socket)
t.Setenv("SPEECHD_SOCKET", "")
uid := os.Geteuid()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
failed := make(chan struct{}, 1)
result := make(chan error, 1)
var client Transport
go func() {
result <- audio.Retry(ctx, func(ctx context.Context) error {
var err error
client, err = Open_driver_context(ctx, "speech-dispatcher", Options{})
if err != nil {
select {
case failed <- struct{}{}:
default:
}
}
return err
}, audio.Temporary)
}()
select {
case <-failed:
case <-ctx.Done():
t.Fatal("initial absent daemon was not detected")
}
listener, err := net.Listen("unix", socket)
if err != nil {
t.Fatal(err)
}
server := new_fake_server_listener(t, false, listener)
if err := <-result; err != nil {
t.Fatal(err)
}
defer client.Close()
cancel()
if os.Geteuid() != uid {
t.Fatal("startup changed reader credentials")
}
if err := client.Say("late daemon ready"); err != nil {
t.Fatal(err)
}
wait_event(t, server.events, "say:late daemon ready")
}
func TestClientReconnectsAfterPrivateServerDropsBothConnections(t *testing.T) {
server := new_fake_server(t, false)
client, err := Open(Options{Socket: server.path})
if err != nil {
t.Fatal(err)
}
service := New(client)
defer service.Close()
service.Configure(Parameters{Rate: 30, Pitch: 10, Volume: 70})
service.Say("before disconnect")
wait_event(t, server.events, "say:before disconnect")
server.mu.Lock()
for _, conn := range server.connections {
conn.Close()
}
server.mu.Unlock()
service.Say("failed request")
deadline := time.NewTimer(time.Second)
defer deadline.Stop()
tick := time.NewTicker(time.Millisecond)
defer tick.Stop()
for {
service.mu.Lock()
service.transportMu.RLock()
restored := service.transport != nil && service.transport != client && !service.recovering
service.transportMu.RUnlock()
service.mu.Unlock()
if restored {
break
}
select {
case err := <-service.Errors():
t.Fatal("reconnection failed", err)
case <-deadline.C:
t.Fatal("both Speech Dispatcher sockets were not restored")
case <-tick.C:
}
}
service.Say("after reconnect")
wait_event(t, server.events, "say:after reconnect")
}
+111 -18
View File
@@ -18,7 +18,11 @@
package speech
import (
"context"
"errors"
"log/slog"
"git.stormux.org/storm/fenrir/internal/audio"
"strings"
"sync"
"time"
@@ -39,6 +43,11 @@ type Transport interface {
type Service struct {
transport Transport
transportMu sync.RWMutex
transportEpoch uint64
faults chan transportFailure
recoveryContext context.Context
stopRecovery context.CancelFunc
recovering bool
replacements []replacement
mu sync.Mutex
dispatch sync.Mutex
@@ -52,6 +61,11 @@ type Service struct {
workers sync.WaitGroup
}
type transportFailure struct {
epoch uint64
err error
}
type replacement struct {
transport Transport
done chan error
@@ -82,7 +96,8 @@ type utterance struct {
}
func New(transport Transport) *Service {
s := &Service{transport: transport, wake: make(chan struct{}, 1), cancelWake: make(chan struct{}, 1), done: make(chan struct{}), errors: make(chan error, 1)}
ctx, cancel := context.WithCancel(context.Background())
s := &Service{recoveryContext: ctx, stopRecovery: cancel, faults: make(chan transportFailure, 1), transport: transport, wake: make(chan struct{}, 1), cancelWake: make(chan struct{}, 1), done: make(chan struct{}), errors: make(chan error, 1)}
s.workers.Add(2)
go s.deliver()
go s.cancel()
@@ -143,7 +158,7 @@ func (s *Service) enqueue(item utterance) {
}
s.mu.Lock()
defer s.mu.Unlock()
if s.closed || s.muted {
if s.closed || s.muted || s.recovering {
return
}
item.queuedAt = time.Now()
@@ -189,6 +204,72 @@ func (s *Service) fail(err error) {
}
}
// Recovery belongs to the delivery worker. Admission and cancellation never
// wait for a new daemon, and speech from the outage is never replayed.
func (s *Service) recover_transport(problem error, parameters *Parameters) bool {
reopener, ok := s.transport.(interface {
Reopen(context.Context) (Transport, error)
})
if !ok || !audio.Temporary(problem) {
s.fail(problem)
return false
}
s.dispatch.Lock()
defer s.dispatch.Unlock()
s.mu.Lock()
s.recovering = true
s.queue = nil
s.mu.Unlock()
s.transportMu.Lock()
old := s.transport
s.transport = nil
s.transportEpoch++
s.transportMu.Unlock()
old.Close()
slog.Warn("speech connection lost; reconnecting", "error", problem, "at", time.Now())
ctx, cancel := context.WithTimeout(s.recoveryContext, audio.RecoveryTimeout)
defer cancel()
var candidate Transport
err := audio.Retry(ctx, func(ctx context.Context) error {
var err error
candidate, err = reopener.Reopen(ctx)
if err == nil && parameters != nil {
if client, ok := candidate.(*Client); ok {
stop := context.AfterFunc(ctx, client.abort)
err = candidate.Configure(*parameters)
stop()
} else {
err = candidate.Configure(*parameters)
}
}
if err != nil && candidate != nil {
candidate.Close()
candidate = nil
}
return err
}, audio.Temporary)
err = errors.Join(err, ctx.Err())
s.mu.Lock()
if err != nil || s.closed || ctx.Err() != nil {
closed := s.closed
s.mu.Unlock()
if candidate != nil {
candidate.Close()
}
if err != nil && !closed {
s.fail(errors.Join(errors.New("speech recovery failed"), err))
}
return false
}
s.transportMu.Lock()
s.transport = candidate
s.transportMu.Unlock()
s.recovering = false
s.mu.Unlock()
slog.Info("speech connection restored", "at", time.Now())
return true
}
func (s *Service) deliver() {
defer s.workers.Done()
normalPitch := 0
@@ -198,6 +279,10 @@ func (s *Service) deliver() {
case <-s.done:
return
case <-s.wake:
case fault := <-s.faults:
if fault.epoch == s.transportEpoch && !s.recover_transport(fault.err, currentParameters) {
return
}
}
for {
s.mu.Lock()
@@ -211,8 +296,9 @@ func (s *Service) deliver() {
err := s.transport.Configure(parameters)
s.dispatch.Unlock()
if err != nil {
s.fail(err)
return
if !s.recover_transport(err, currentParameters) {
return
}
}
continue
}
@@ -233,6 +319,7 @@ func (s *Service) deliver() {
s.transportMu.Lock()
old := s.transport
s.transport = request.transport
s.transportEpoch++
s.transportMu.Unlock()
s.mu.Unlock()
old.Close()
@@ -262,8 +349,9 @@ func (s *Service) deliver() {
diagnostic.Timing("speech send", started, "generation", generation)
s.dispatch.Unlock()
if err != nil {
s.fail(err)
return
if !s.recover_transport(err, currentParameters) {
return
}
}
}
}
@@ -293,7 +381,17 @@ func (s *Service) say(item utterance, normalPitch int, normalParameters *Paramet
func (s *Service) cancel_transport() error {
s.transportMu.RLock()
defer s.transportMu.RUnlock()
return s.transport.Cancel()
if s.transport == nil {
return nil
}
err := s.transport.Cancel()
if err != nil {
select {
case s.faults <- transportFailure{epoch: s.transportEpoch, err: err}:
default:
}
}
return err
}
func (s *Service) cancel() {
@@ -320,21 +418,13 @@ func (s *Service) cancel() {
if !s.dispatch.TryLock() {
// Stop audible speech even while a stale send is blocked. A
// second cancel after that send finishes closes the late-send race.
if err := s.cancel_transport(); err != nil {
s.fail(err)
return
}
_ = s.cancel_transport()
s.dispatch.Lock()
}
// With dispatch held, no send can race this cancel. The usual
// idle-send path needs only this one acknowledgement.
err := s.cancel_transport()
_ = s.cancel_transport()
diagnostic.Timing("speech cancel barrier", started, "generation", generation)
if err != nil {
s.dispatch.Unlock()
s.fail(err)
return
}
s.mu.Lock()
// Every intervening interrupt is covered: fresh sends have been
// gated throughout. Coalesce them instead of canceling again.
@@ -354,6 +444,7 @@ func (s *Service) Close() {
return
}
s.closed = true
s.stopRecovery()
s.queue = nil
close(s.done)
pending := s.replacements
@@ -364,7 +455,9 @@ func (s *Service) Close() {
request.done <- errors.New("speech service closed")
}
s.transportMu.RLock()
s.transport.Close()
if s.transport != nil {
s.transport.Close()
}
s.transportMu.RUnlock()
s.workers.Wait()
}