storm/barnard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: add jitter buffer, sequence tracking, and mic error reporting

Browse Source 
Add a jitter buffer in OnAudioStream that collects incoming audio packets,
sorts by sequence number, and releases them in order after a small initial
delay (3 packets, ~30ms). This prevents out-of-order playback from network
jitter and reordered UDP packets.

Add a Sequence field to AudioPacket so the jitter buffer can reorder by
sequence number. PLC frames generated by dispatchPLC now also carry the
correct sequence number so they integrate with the jitter buffer.

Add microphone capture error reporting: when CaptureSamples returns fewer
bytes than expected, report the failure to the user via a callback. The
error is reported once when the failure starts and again when the mic
recovers. Wire this up in client.go to display status messages.
This commit is contained in:
Brandon McGinty (deepseek)
2026-08-08 21:02:33 -04:00
committed by Brandon McGinty
parent 9dd0137975
commit 0858cd3542
4 changed files with 221 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client.go
+7
View File
@@ -53,6 +53,13 @@ func (b *Barnard) connect(reconnect bool) bool {
b.Stream = stream
b.Stream.AttachStream(b.Client)
b.Stream.SetNoiseProcessor(b.NoiseSuppressor)
b.Stream.SetErrorFunc(func(err error) {
if err != nil {
b.AddOutputLine(fmt.Sprintf("Microphone: %s", err.Error()))
} else {
b.AddOutputLine("Microphone: recovered")
}
})
// Initialize stereo encoder for file playback
b.Client.AudioEncoderStereo = opus.NewStereoEncoder()
gumble/gumble/audio.go
+4
View File
@@ -84,6 +84,10 @@ type AudioPacket struct {
Sender *User
Target *VoiceTarget
// Sequence is the UDP audio packet sequence number, used by the
// jitter buffer to reorder packets.
Sequence int64
AudioBuffer
HasPosition bool
gumble/gumble/handlers.go
+5
View File
@@ -162,6 +162,7 @@ func (c *Client) handleUDPTunnel(buffer []byte) error {
Target: &VoiceTarget{
ID: uint32(audioTarget),
},
Sequence: seq,
AudioBuffer: AudioBuffer(pcm),
}
@@ -181,6 +182,7 @@ func (c *Client) handleUDPTunnel(buffer []byte) error {
// dispatchPLC generates a Packet Loss Concealment frame from the decoder
// and dispatches it to all audio listeners for the given user.
// seq is the expected sequence number for the concealed frame.
func (c *Client) dispatchPLC(user *User, audioTarget byte, decoder AudioDecoder) {
// Feed empty data to the decoder to trigger Opus PLC, which
// produces a concealed frame bridging the gap.
@@ -191,10 +193,13 @@ func (c *Client) dispatchPLC(user *User, audioTarget byte, decoder AudioDecoder)
decoder.Reset()
return
}
seq := user.audioSequence + 1
user.audioSequence = seq
event := AudioPacket{
Client: c,
Sender: user,
Target: &VoiceTarget{ID: uint32(audioTarget)},
Sequence: seq,
AudioBuffer: AudioBuffer(pcm),
}
c.dispatchAudio(user, &event)
gumble/gumbleopenal/stream.go
+205 -100
View File
@@ -74,6 +74,7 @@ type Stream struct {
micAGCRight *audio.AGC
filePlayer FilePlayer
recorderMu sync.RWMutex
errorFunc func(error) // called on capture errors
recorder Recorder
}
@@ -154,6 +155,12 @@ func (s *Stream) GetFilePlayer() FilePlayer {
return s.filePlayer
}
// SetErrorFunc sets a callback that is invoked when the microphone
// capture device fails to provide audio data.
func (s *Stream) SetErrorFunc(f func(error)) {
s.errorFunc = f
}
func (s *Stream) SetRecorder(recorder Recorder) {
s.recorderMu.Lock()
defer s.recorderMu.Unlock()
@@ -261,118 +268,90 @@ func (s *Stream) OnAudioStream(e *gumble.AudioStreamEvent) {
var raw [maxBufferSize]byte
// Jitter buffer: collects incoming packets, reorders by
// sequence number, and releases them in order after a small
// initial delay (3-5 packets / 30-50ms).
const (
jitterMinPackets = 3 // minimum packets before playout starts
jitterMaxPackets = 10 // maximum buffered packets before dropping oldest
)
var jitterBuf []*gumble.AudioPacket
var jitterNextSeq int64
var jitterInit bool
// insertSorted inserts a packet into the jitter buffer sorted
// by sequence number.
insertSorted := func(p *gumble.AudioPacket) {
// Drop if we already have too many (protect against memory bloat)
if len(jitterBuf) >= jitterMaxPackets {
return
}
// Find insertion point (ascending sequence order)
i := 0
for i < len(jitterBuf) && jitterBuf[i].Sequence < p.Sequence {
i++
}
// Don't insert duplicates
if i < len(jitterBuf) && jitterBuf[i].Sequence == p.Sequence {
return
}
jitterBuf = append(jitterBuf, nil)
copy(jitterBuf[i+1:], jitterBuf[i:])
jitterBuf[i] = p
}
// popNext removes and returns the packet with the expected next
// sequence number, or nil if not yet available.
popNext := func() *gumble.AudioPacket {
if len(jitterBuf) == 0 || jitterBuf[0].Sequence != jitterNextSeq {
return nil
}
p := jitterBuf[0]
jitterBuf = jitterBuf[1:]
jitterNextSeq = p.Sequence + 1
return p
}
for packet := range e.C {
// Skip processing if user is locally muted
if e.User.LocallyMuted() {
continue
}
samples := len(packet.AudioBuffer)
if samples > cap(raw)/2 {
// Insert into jitter buffer
insertSorted(packet)
// Initialize the expected sequence on first packet
if !jitterInit {
jitterNextSeq = jitterBuf[0].Sequence
jitterInit = true
}
// Only start playing when we have enough buffered packets
if len(jitterBuf) < jitterMinPackets {
continue
}
boost := e.User.Boost()
userVolume := e.User.Volume()
recorder := s.getRecorder()
var recordBuffer []int16
recordPtr := 0
if recorder != nil {
recordBuffer = make([]int16, len(packet.AudioBuffer)*gumble.AudioChannels)
}
// Check if sample count suggests stereo data
isStereo := samples > gumble.AudioDefaultFrameSize && samples%2 == 0
format := openal.FormatMono16
if isStereo {
format = openal.FormatStereo16
samples = samples / 2
}
rawPtr := 0
if isStereo {
// Process stereo samples as pairs
for i := 0; i < samples*2; i += 2 {
// Process left channel with saturation protection
sample := packet.AudioBuffer[i]
if boost > 1 {
boosted := int32(sample) * int32(boost)
if boosted > 32767 {
sample = 32767
} else if boosted < -32767 {
sample = -32767
} else {
sample = int16(boosted)
}
}
if recorder != nil {
recordBuffer[recordPtr] = scaleForRecording(sample, userVolume)
recordPtr++
}
binary.LittleEndian.PutUint16(raw[rawPtr:], uint16(sample))
rawPtr += 2
// Process right channel with saturation protection
sample = packet.AudioBuffer[i+1]
if boost > 1 {
boosted := int32(sample) * int32(boost)
if boosted > 32767 {
sample = 32767
} else if boosted < -32767 {
sample = -32767
} else {
sample = int16(boosted)
}
}
if recorder != nil {
recordBuffer[recordPtr] = scaleForRecording(sample, userVolume)
recordPtr++
}
binary.LittleEndian.PutUint16(raw[rawPtr:], uint16(sample))
rawPtr += 2
}
} else {
// Process mono samples with saturation protection
for i := 0; i < samples; i++ {
sample := packet.AudioBuffer[i]
if boost > 1 {
boosted := int32(sample) * int32(boost)
if boosted > 32767 {
sample = 32767
} else if boosted < -32767 {
sample = -32767
} else {
sample = int16(boosted)
}
}
if recorder != nil {
recordSample := scaleForRecording(sample, userVolume)
recordBuffer[recordPtr] = recordSample
recordBuffer[recordPtr+1] = recordSample
recordPtr += 2
}
binary.LittleEndian.PutUint16(raw[rawPtr:], uint16(sample))
rawPtr += 2
// Drain all packets that are ready (in sequence order)
for {
pkt := popNext()
if pkt == nil {
break
}
emptyBufs = s.processAudioPacket(pkt, e.User, &source, emptyBufs, &raw, reclaim)
}
if recorder != nil && recordPtr > 0 {
recorder.RecordAudioFrame(e.User.Session, recordBuffer[:recordPtr])
}
// Drain remaining buffered packets on stream close
for len(jitterBuf) > 0 {
pkt := popNext()
if pkt == nil {
// Gap in sequence at end; skip
jitterNextSeq = jitterBuf[0].Sequence
pkt = popNext()
}
reclaim()
if len(emptyBufs) == 0 {
continue
}
last := len(emptyBufs) - 1
buffer := emptyBufs[last]
emptyBufs = emptyBufs[:last]
buffer.SetData(format, raw[:rawPtr], gumble.AudioSampleRate)
source.QueueBuffer(buffer)
if source.State() != openal.Playing {
source.Play()
if pkt != nil {
emptyBufs = s.processAudioPacket(pkt, e.User, &source, emptyBufs, &raw, reclaim)
}
}
reclaim()
@@ -381,6 +360,120 @@ func (s *Stream) OnAudioStream(e *gumble.AudioStreamEvent) {
}(e)
}
// processAudioPacket decodes and queues a single audio packet for playback.
// Returns the updated emptyBufs slice after consuming a buffer.
func (s *Stream) processAudioPacket(packet *gumble.AudioPacket, user *gumble.User, source *openal.Source, emptyBufs openal.Buffers, raw *[maxBufferSize]byte, reclaim func()) openal.Buffers {
samples := len(packet.AudioBuffer)
if samples > cap(*raw)/2 {
return emptyBufs
}
boost := user.Boost()
userVolume := user.Volume()
recorder := s.getRecorder()
var recordBuffer []int16
recordPtr := 0
if recorder != nil {
recordBuffer = make([]int16, len(packet.AudioBuffer)*gumble.AudioChannels)
}
// Check if sample count suggests stereo data
isStereo := samples > gumble.AudioDefaultFrameSize && samples%2 == 0
format := openal.FormatMono16
if isStereo {
format = openal.FormatStereo16
samples = samples / 2
}
rawPtr := 0
if isStereo {
// Process stereo samples as pairs
for i := 0; i < samples*2; i += 2 {
// Process left channel with saturation protection
sample := packet.AudioBuffer[i]
if boost > 1 {
boosted := int32(sample) * int32(boost)
if boosted > 32767 {
sample = 32767
} else if boosted < -32767 {
sample = -32767
} else {
sample = int16(boosted)
}
}
if recorder != nil {
recordBuffer[recordPtr] = scaleForRecording(sample, userVolume)
recordPtr++
}
binary.LittleEndian.PutUint16((*raw)[rawPtr:], uint16(sample))
rawPtr += 2
// Process right channel with saturation protection
sample = packet.AudioBuffer[i+1]
if boost > 1 {
boosted := int32(sample) * int32(boost)
if boosted > 32767 {
sample = 32767
} else if boosted < -32767 {
sample = -32767
} else {
sample = int16(boosted)
}
}
if recorder != nil {
recordBuffer[recordPtr] = scaleForRecording(sample, userVolume)
recordPtr++
}
binary.LittleEndian.PutUint16((*raw)[rawPtr:], uint16(sample))
rawPtr += 2
}
} else {
// Process mono samples with saturation protection
for i := 0; i < samples; i++ {
sample := packet.AudioBuffer[i]
if boost > 1 {
boosted := int32(sample) * int32(boost)
if boosted > 32767 {
sample = 32767
} else if boosted < -32767 {
sample = -32767
} else {
sample = int16(boosted)
}
}
if recorder != nil {
recordSample := scaleForRecording(sample, userVolume)
recordBuffer[recordPtr] = recordSample
recordBuffer[recordPtr+1] = recordSample
recordPtr += 2
}
binary.LittleEndian.PutUint16((*raw)[rawPtr:], uint16(sample))
rawPtr += 2
}
}
if recorder != nil && recordPtr > 0 {
recorder.RecordAudioFrame(user.Session, recordBuffer[:recordPtr])
}
reclaim()
if len(emptyBufs) == 0 {
return emptyBufs
}
last := len(emptyBufs) - 1
buffer := emptyBufs[last]
emptyBufs[last] = 0
emptyBufs = emptyBufs[:last]
buffer.SetData(format, (*raw)[:rawPtr], gumble.AudioSampleRate)
source.QueueBuffer(buffer)
if source.State() != openal.Playing {
source.Play()
}
return emptyBufs
}
func (s *Stream) sourceRoutine(inputDevice *string) {
interval := s.client.Config.AudioInterval
frameSize := s.client.Config.AudioFrameSize()
@@ -407,6 +500,7 @@ func (s *Stream) sourceRoutine(inputDevice *string) {
outgoing := s.client.AudioOutgoing()
defer close(outgoing)
var micFailed bool
for {
select {
case <-stop:
@@ -420,6 +514,12 @@ func (s *Stream) sourceRoutine(inputDevice *string) {
buff := s.deviceSource.CaptureSamples(uint32(frameSize))
if len(buff) == sampleCount*2 {
hasMicInput = true
if micFailed {
micFailed = false
if s.errorFunc != nil {
s.errorFunc(nil) // nil signals recovery
}
}
for i := 0; i < sampleCount; i++ {
sample := int16(binary.LittleEndian.Uint16(buff[i*2:]))
vol := s.GetMicVolume()
@@ -434,6 +534,11 @@ func (s *Stream) sourceRoutine(inputDevice *string) {
} else {
s.processStereoSamples(int16Buffer, frameSize)
}
} else if !micFailed {
micFailed = true
if s.errorFunc != nil {
s.errorFunc(ErrMic)
}
}
// Mix with or use file audio if playing