Checkpoint audio-only Skald fork work

This commit is contained in:
Storm Dragon
2026-05-18 13:06:57 -04:00
parent a8ada950d5
commit 965347cad4
48 changed files with 1080 additions and 3651 deletions
+23 -111
View File
@@ -230,68 +230,12 @@ func (down *rtpDownTrack) Write(buf []byte) (int, error) {
return 0, err
}
layer := down.getLayerInfo()
if flags.Tid > layer.maxTid || flags.Sid > layer.maxSid {
if flags.Tid > layer.maxTid {
// increase eagerly if this is the first time we
// see a given layer
if layer.tid == layer.maxTid {
layer.wantedTid = flags.Tid
layer.tid = flags.Tid
}
layer.maxTid = flags.Tid
}
if flags.Sid > layer.maxSid {
if layer.sid == layer.maxSid && !layer.limitSid {
layer.wantedSid = flags.Sid
layer.sid = flags.Sid
}
layer.maxSid = flags.Sid
}
down.setLayerInfo(layer)
down.adjustLayer()
layer = down.getLayerInfo()
}
if flags.Start && (layer.tid != layer.wantedTid) {
if flags.Keyframe {
layer.tid = layer.wantedTid
down.setLayerInfo(layer)
} else if layer.wantedTid < layer.tid {
layer.tid = layer.wantedTid
down.setLayerInfo(layer)
} else if flags.TidUpSync && flags.Tid <= layer.wantedTid {
layer.tid = flags.Tid
down.setLayerInfo(layer)
}
}
if flags.Start && (layer.sid != layer.wantedSid) {
if flags.Keyframe {
layer.sid = layer.wantedSid
down.setLayerInfo(layer)
} else {
down.remote.RequestKeyframe()
}
}
if flags.Tid > layer.tid || flags.Sid > layer.sid ||
(flags.Sid < layer.sid && flags.SidNonReference) {
ok := down.packetmap.Drop(flags.Seqno, flags.Pid)
if ok {
return 0, nil
}
}
ok, newseqno, piddelta := down.packetmap.Map(flags.Seqno, flags.Pid)
if !ok {
return 0, nil
}
setMarker := flags.Sid == layer.sid && flags.End && !flags.Marker
if !setMarker && newseqno == flags.Seqno && piddelta == 0 {
if newseqno == flags.Seqno && piddelta == 0 {
return down.write(buf)
}
@@ -300,7 +244,7 @@ func (down *rtpDownTrack) Write(buf []byte) (int, error) {
buf2 := ibuf2.([]byte)
n := copy(buf2, buf)
err = codecs.RewritePacket(codec, buf2[:n], setMarker, newseqno, piddelta)
err = codecs.RewritePacket(codec, buf2[:n], false, newseqno, piddelta)
if err != nil {
return 0, err
}
@@ -420,7 +364,6 @@ type trackActionKind int
const (
trackActionAdd trackActionKind = iota
trackActionDel
trackActionKeyframe
)
type trackAction struct {
@@ -454,11 +397,6 @@ func (up *rtpUpTrack) AddLocal(local conn.DownTrack) error {
return nil
}
func (up *rtpUpTrack) RequestKeyframe() error {
up.action(trackActionKeyframe, nil)
return nil
}
func (up *rtpUpTrack) DelLocal(local conn.DownTrack) bool {
up.mu.Lock()
for i, l := range up.local {
@@ -598,7 +536,7 @@ func (up *rtpUpConnection) flushICECandidates() error {
return err
}
// pushConnNow pushes a connection to all of the clients in a group
// pushConnNow pushes a connection to all of the clients in a hall
func pushConnNow(up *rtpUpConnection, g *hall.Hall, cs []hall.Client) {
up.mu.Lock()
up.pushed = true
@@ -650,8 +588,13 @@ func newUpConn(c hall.Client, id string, label string, offer string) (*rtpUpConn
}
for _, m := range o.MediaDescriptions {
tpe := webrtc.NewRTPCodecType(m.MediaName.Media)
if tpe == webrtc.RTPCodecTypeVideo {
log.Printf("Rejecting incoming video media section")
continue
}
_, err = pc.AddTransceiverFromKind(
webrtc.NewRTPCodecType(m.MediaName.Media),
tpe,
webrtc.RTPTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionRecvonly,
},
@@ -665,6 +608,10 @@ func newUpConn(c hall.Client, id string, label string, offer string) (*rtpUpConn
up := &rtpUpConnection{id: id, client: c, label: label, pc: pc}
pc.OnTrack(func(remote *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
if remote.Kind() == webrtc.RTPCodecTypeVideo {
log.Printf("Rejecting incoming video track")
return
}
up.mu.Lock()
track := &rtpUpTrack{
@@ -698,26 +645,16 @@ func newUpConn(c hall.Client, id string, label string, offer string) (*rtpUpConn
var ErrUnsupportedFeedback = errors.New("unsupported feedback type")
var ErrRateLimited = errors.New("rate limited")
func (track *rtpUpTrack) sendPLI() error {
if !track.hasRtcpFb("nack", "pli") {
return ErrUnsupportedFeedback
}
return sendPLI(track.conn.pc, track.track.SSRC())
}
func sendPLI(pc *webrtc.PeerConnection, ssrc webrtc.SSRC) error {
return pc.WriteRTCP([]rtcp.Packet{
&rtcp.PictureLossIndication{MediaSSRC: uint32(ssrc)},
})
}
func (track *rtpUpTrack) sendNACK(first uint16, bitmap uint16) error {
if !track.hasRtcpFb("nack", "") {
return ErrUnsupportedFeedback
}
err := sendNACKs(track.conn.pc, track.track.SSRC(),
[]rtcp.NackPair{{first, rtcp.PacketBitmap(bitmap)}},
[]rtcp.NackPair{{
PacketID: first,
LostPackets: rtcp.PacketBitmap(bitmap),
}},
)
if err == nil {
track.cache.Expect(1 + bits.OnesCount16(bitmap))
@@ -744,7 +681,10 @@ func (track *rtpUpTrack) sendNACKs(seqnos []uint16) error {
}
var f, b uint16
f, b, seqnos = packetcache.ToBitmap(seqnos)
nacks = append(nacks, rtcp.NackPair{f, rtcp.PacketBitmap(b)})
nacks = append(nacks, rtcp.NackPair{
PacketID: f,
LostPackets: rtcp.PacketBitmap(b),
})
}
err := sendNACKs(track.conn.pc, track.track.SSRC(), nacks)
if err == nil {
@@ -814,7 +754,7 @@ func rtcpUpListener(track *rtpUpTrack) {
for {
firstSR := false
n, _, err := track.receiver.ReadSimulcast(buf, track.track.RID())
n, _, err := track.receiver.Read(buf)
if err != nil {
if err != io.EOF && err != io.ErrClosedPipe {
log.Printf("Read RTCP: %v", err)
@@ -1147,7 +1087,6 @@ func (track *rtpDownTrack) updateRate(loss uint8, now uint64) {
}
func rtcpDownListener(track *rtpDownTrack) {
lastFirSeqno := uint8(0)
buf := make([]byte, 1500)
@@ -1165,40 +1104,18 @@ func rtcpDownListener(track *rtpDownTrack) {
continue
}
adjust := false
jiffies := rtptime.Jiffies()
for _, p := range ps {
switch p := p.(type) {
case *rtcp.PictureLossIndication:
track.remote.RequestKeyframe()
case *rtcp.FullIntraRequest:
found := false
var seqno uint8
for _, entry := range p.FIR {
if entry.SSRC == uint32(track.ssrc) {
found = true
seqno = entry.SequenceNumber
break
}
}
if !found {
log.Printf("Misdirected FIR")
continue
}
if seqno != lastFirSeqno {
track.remote.RequestKeyframe()
}
case *rtcp.ReceiverEstimatedMaximumBitrate:
rate := uint64(p.Bitrate + 0.5)
track.maxREMBBitrate.Set(rate, jiffies)
adjust = true
case *rtcp.ReceiverReport:
for _, r := range p.Reports {
if r.SSRC == uint32(track.ssrc) {
handleReport(track, r, jiffies)
adjust = true
}
}
case *rtcp.SenderReport:
@@ -1211,9 +1128,7 @@ func rtcpDownListener(track *rtpDownTrack) {
gotNACK(track, p)
}
}
if adjust {
track.adjustLayer()
}
}
}
@@ -1245,9 +1160,6 @@ func handleReport(track *rtpDownTrack, report rtcp.ReceptionReport, jiffies uint
}
func minPacketCache(track *webrtc.TrackRemote) int {
if track.Kind() == webrtc.RTPCodecTypeVideo {
return 128
}
return 24
}
+2 -30
View File
@@ -3,12 +3,9 @@ package rtpconn
import (
"io"
"log"
"time"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
"git.stormux.org/storm/skald/codecs"
"git.stormux.org/storm/skald/packetcache"
"git.stormux.org/storm/skald/rtptime"
)
@@ -20,12 +17,7 @@ func readLoop(track *rtpUpTrack) {
close(track.readerDone)
}()
isvideo := track.track.Kind() == webrtc.RTPCodecTypeVideo
codec := track.track.Codec()
sendNACK := track.hasRtcpFb("nack", "")
sendPLI := track.hasRtcpFb("nack", "pli")
var kfNeeded bool
var kfRequested time.Time
buf := make([]byte, packetcache.BufSize)
var packet rtp.Packet
for {
@@ -46,8 +38,6 @@ func readLoop(track *rtpUpTrack) {
err,
)
}
case trackActionKeyframe:
kfNeeded = true
default:
log.Printf("Unknown action")
}
@@ -72,10 +62,6 @@ func readLoop(track *rtpUpTrack) {
track.jitter.Accumulate(packet.Timestamp)
kf, kfKnown := codecs.Keyframe(codec.MimeType, &packet)
if kf || !kfKnown {
kfNeeded = false
}
if packet.Extension {
packet.Extension = false
packet.Extensions = nil
@@ -88,7 +74,7 @@ func readLoop(track *rtpUpTrack) {
first, index := track.cache.Store(
packet.SequenceNumber, packet.Timestamp,
kf, packet.Marker, buf[:bytes],
false, packet.Marker, buf[:bytes],
)
_, rate := track.rate.Estimate()
@@ -131,20 +117,6 @@ func readLoop(track *rtpUpTrack) {
}
writers.write(packet.SequenceNumber, index, delay,
isvideo, packet.Marker)
now := time.Now()
if kfNeeded && now.Sub(kfRequested) > time.Second/2 {
if sendPLI {
err := track.sendPLI()
if err != nil {
log.Printf("sendPLI: %v", err)
kfNeeded = false
}
} else {
kfNeeded = false
}
kfRequested = now
}
packet.Marker)
}
}
+8 -24
View File
@@ -105,7 +105,7 @@ func (wp *rtpWriterPool) close() {
}
// write writes a packet stored in the packet cache to all local tracks
func (wp *rtpWriterPool) write(seqno uint16, index uint16, delay uint32, isvideo bool, marker bool) {
func (wp *rtpWriterPool) write(seqno uint16, index uint16, delay uint32, marker bool) {
pi := packetIndex{seqno, index}
var dead []*rtpWriter
@@ -127,15 +127,7 @@ func (wp *rtpWriterPool) write(seqno uint16, index uint16, delay uint32, isvideo
// the writer is dead.
dead = append(dead, w)
default:
// the writer is congested
if isvideo {
// drop until the end of the frame
if !marker {
w.drop = 7
}
continue
}
// audio, try again with a delay
// the writer is congested, try again with a delay
d := delay / uint32(2*len(wp.writers))
timer := time.NewTimer(rtptime.ToDuration(
int64(d), rtptime.JiffiesPerSec,
@@ -256,20 +248,12 @@ func rtpWriterLoop(writer *rtpWriter, track *rtpUpTrack) {
}
last, foundLast := track.cache.Last()
kf, foundKf := track.cache.Keyframe()
if foundLast && foundKf {
if last-kf < 40 { // modulo 2^16
go sendSequence(
kf, last,
action.track,
track.cache,
)
} else {
track.RequestKeyframe()
}
} else {
// no keyframe yet, one should
// arrive soon. Do nothing.
if foundLast {
go sendSequence(
last, last,
action.track,
track.cache,
)
}
} else {
found := false
+52 -88
View File
@@ -10,7 +10,6 @@ import (
"maps"
"net"
"os"
"strings"
"sync"
"time"
@@ -109,9 +108,9 @@ func (c *webClient) SetPermissions(perms []string) {
c.permissions = perms
}
func (c *webClient) PushClient(group, kind, id string, username string, perms []string, data map[string]interface{}) error {
func (c *webClient) PushClient(hall, kind, id string, username string, perms []string, data map[string]interface{}) error {
c.action(pushClientAction{
group, kind, id, username, perms, data,
hall, kind, id, username, perms, data,
})
return nil
}
@@ -132,7 +131,7 @@ type clientMessage struct {
Permissions []string `json:"permissions,omitempty"`
Status *hall.Status `json:"status,omitempty"`
Data map[string]interface{} `json:"data,omitempty"`
Group string `json:"group,omitempty"`
Hall string `json:"hall,omitempty"`
Value interface{} `json:"value,omitempty"`
NoEcho bool `json:"noecho,omitempty"`
Time string `json:"time,omitempty"`
@@ -385,11 +384,7 @@ func addDownTrackUnlocked(conn *rtpDownConnection, remoteTrack *rtpUpTrack) erro
// replace the RTCP feedback types with the ones we understand
remoteCodec := remoteTrack.Codec()
if strings.HasPrefix(strings.ToLower(remoteCodec.MimeType), "video/") {
remoteCodec.RTCPFeedback = hall.VideoRTCPFeedback
} else {
remoteCodec.RTCPFeedback = hall.AudioRTCPFeedback
}
remoteCodec.RTCPFeedback = hall.AudioRTCPFeedback
local, err := webrtc.NewTrackLocalStaticRTP(
remoteCodec, id, msid,
@@ -727,7 +722,7 @@ func parseRequested(r interface{}) (map[string][]string, error) {
func (c *webClient) setRequested(requested map[string][]string) error {
if c.hall == nil {
return errors.New("attempted to request with no group joined")
return errors.New("attempted to request with no hall joined")
}
c.requested = requested
@@ -761,60 +756,27 @@ func requestedTracks(c *webClient, requested []string, tracks []conn.UpTrack) ([
if len(requested) == 0 {
return nil, false
}
var audio, video, videoLow bool
var audio bool
for _, s := range requested {
switch s {
case "audio":
audio = true
case "video":
video = true
case "video-low":
videoLow = true
default:
log.Printf("client requested unknown value %v", s)
}
}
find := func(kind webrtc.RTPCodecType, last bool) (conn.UpTrack, int) {
var track conn.UpTrack
count := 0
var ts []conn.UpTrack
if audio {
for _, t := range tracks {
if t.Kind() != kind {
continue
}
track = t
count++
if !last {
if t.Kind() == webrtc.RTPCodecTypeAudio {
ts = append(ts, t)
break
}
}
return track, count
}
var ts []conn.UpTrack
limitSid := false
if audio {
t, _ := find(webrtc.RTPCodecTypeAudio, false)
if t != nil {
ts = append(ts, t)
}
}
if video {
t, _ := find(webrtc.RTPCodecTypeVideo, false)
if t != nil {
ts = append(ts, t)
}
} else if videoLow {
t, count := find(webrtc.RTPCodecTypeVideo, true)
if t != nil {
ts = append(ts, t)
}
if count < 2 {
limitSid = true
}
}
return ts, limitSid
return ts, false
}
func (c *webClient) PushConn(g *hall.Hall, id string, up conn.Up, tracks []conn.UpTrack, replace string) error {
@@ -914,7 +876,7 @@ type connectionFailedAction struct {
}
type pushClientAction struct {
group string
hall string
kind string
id string
username string
@@ -929,8 +891,8 @@ type changePermissionsAction struct {
type permissionsChangedAction struct{}
type joinedAction struct {
group string
kind string
hall string
kind string
}
type kickAction struct {
@@ -972,7 +934,7 @@ func clientLoop(c *webClient, ws *websocket.Conn, versionError bool) error {
read := make(chan interface{}, 1)
go clientReader(ws, read, c.done)
defer leaveGroup(c)
defer leaveHall(c)
readTime := time.Now()
@@ -1109,7 +1071,7 @@ func handleAction(c *webClient, a any) error {
switch a := a.(type) {
case pushConnAction:
if c.hall == nil || c.hall != a.hall {
log.Printf("Got connectsions for wrong group")
log.Printf("Got connectsions for wrong hall")
return nil
}
return pushDownConn(c, a.id, a.conn, a.tracks, a.replace)
@@ -1163,7 +1125,7 @@ func handleAction(c *webClient, a any) error {
}
case pushClientAction:
if a.group != c.hall.Name() {
if a.hall != c.hall.Name() {
log.Printf("got client for wrong hall")
return nil
}
@@ -1181,8 +1143,8 @@ func handleAction(c *webClient, a any) error {
var status *hall.Status
var data map[string]interface{}
var g *hall.Hall
if a.group != "" {
g = hall.Get(a.group)
if a.hall != "" {
g = hall.Get(a.hall)
if g != nil {
s := g.Status(true, nil)
status = &s
@@ -1194,7 +1156,7 @@ func handleAction(c *webClient, a any) error {
err := c.write(clientMessage{
Type: "joined",
Kind: a.kind,
Group: a.group,
Hall: a.hall,
Username: &username,
Permissions: perms,
Status: status,
@@ -1252,7 +1214,7 @@ func handleAction(c *webClient, a any) error {
case permissionsChangedAction:
g := c.Hall()
if g == nil {
return errors.New("Permissions changed in no group")
return errors.New("Permissions changed in no hall")
}
perms := append([]string(nil), c.permissions...)
status := g.Status(true, nil)
@@ -1260,7 +1222,7 @@ func handleAction(c *webClient, a any) error {
c.write(clientMessage{
Type: "joined",
Kind: "change",
Group: g.Name(),
Hall: g.Name(),
Username: &username,
Permissions: perms,
Status: &status,
@@ -1293,7 +1255,9 @@ func handleAction(c *webClient, a any) error {
}(clients)
case kickAction:
return hall.KickError{
a.id, a.username, a.message,
Id: a.id,
Username: a.username,
Message: a.message,
}
default:
log.Printf("unexpected action %T", a)
@@ -1321,7 +1285,7 @@ func failUpConnection(c *webClient, id string, message string) error {
return nil
}
func leaveGroup(c *webClient) {
func leaveHall(c *webClient) {
if c.hall == nil {
return
}
@@ -1370,8 +1334,8 @@ func (c *webClient) Kick(id string, user *string, message string) error {
return nil
}
func (c *webClient) Joined(group, kind string) error {
c.action(joinedAction{group, kind})
func (c *webClient) Joined(hall, kind string) error {
c.action(joinedAction{hall, kind})
return nil
}
@@ -1402,10 +1366,10 @@ func handleClientMessage(c *webClient, m clientMessage) error {
switch m.Type {
case "join":
if m.Kind == "leave" {
if c.hall == nil || c.hall.Name() != m.Group {
if c.hall == nil || c.hall.Name() != m.Hall {
return hall.UserError("you are not joined")
}
leaveGroup(c)
leaveHall(c)
return nil
}
@@ -1415,11 +1379,11 @@ func handleClientMessage(c *webClient, m clientMessage) error {
if c.hall != nil {
return hall.ProtocolError(
"cannot join multiple groups",
"cannot join multiple halls",
)
}
c.data = m.Data
g, err := hall.AddClient(m.Group, c,
g, err := hall.AddClient(m.Hall, c,
hall.ClientCredentials{
Username: m.Username,
Password: m.Password,
@@ -1438,33 +1402,33 @@ func handleClientMessage(c *webClient, m clientMessage) error {
} else if errors.As(err, &autherr) {
s = "not authorised"
time.Sleep(200 * time.Millisecond)
log.Printf("Join group: %v", err)
log.Printf("Join hall: %v", err)
} else if errors.Is(err, os.ErrNotExist) {
s = "group does not exist"
s = "hall does not exist"
} else if _, ok := err.(hall.UserError); ok {
s = err.Error()
} else {
s = "internal server error"
log.Printf("Join group: %v", err)
log.Printf("Join hall: %v", err)
}
username := c.username
return c.write(clientMessage{
Type: "joined",
Kind: "fail",
Error: e,
Group: m.Group,
Hall: m.Hall,
Username: &username,
Value: s,
})
}
if redirect := g.Description().Redirect; redirect != "" {
// We normally redirect at the HTTP level, but the group
// We normally redirect at the HTTP level, but the hall
// description could have been edited in the meantime.
username := c.username
return c.write(clientMessage{
Type: "joined",
Kind: "redirect",
Group: m.Group,
Hall: m.Hall,
Username: &username,
Value: redirect,
})
@@ -1574,7 +1538,7 @@ func handleClientMessage(c *webClient, m clientMessage) error {
case "chat", "usermessage":
g := c.hall
if g == nil {
return c.error(hall.UserError("join a group first"))
return c.error(hall.UserError("join a hall first"))
}
required := "message"
@@ -1635,10 +1599,10 @@ func handleClientMessage(c *webClient, m clientMessage) error {
}
ccc.write(mm)
}
case "groupaction":
case "hallaction":
g := c.hall
if g == nil {
return c.error(hall.UserError("join a group first"))
return c.error(hall.UserError("join a hall first"))
}
switch m.Kind {
case "clearchat":
@@ -1714,12 +1678,12 @@ func handleClientMessage(c *webClient, m clientMessage) error {
hall.DelClient(disk)
}
}
case "subgroups":
case "subhalls":
if !member("op", c.permissions) {
return c.error(hall.UserError("not authorised"))
}
s := ""
for _, sg := range hall.GetSubGroups(g.Name()) {
for _, sg := range hall.GetSubHalls(g.Name()) {
plural := ""
if sg.Clients > 1 {
plural = "s"
@@ -1773,10 +1737,10 @@ func handleClientMessage(c *webClient, m clientMessage) error {
return terror("error", "client specified token")
}
if tok.Group != c.hall.Name() {
return terror("error", "wrong group in token")
if tok.Hall != c.hall.Name() {
return terror("error", "wrong hall in token")
}
if tok.IncludeSubgroups {
if tok.IncludeSubhalls {
return terror("error",
"hierarchical token not allowed",
)
@@ -1835,8 +1799,8 @@ func handleClientMessage(c *webClient, m clientMessage) error {
if err != nil {
return terror("error", err.Error())
}
if tok.Group != "" || tok.Username != nil ||
tok.IncludeSubgroups ||
if tok.Hall != "" || tok.Username != nil ||
tok.IncludeSubhalls ||
tok.Permissions != nil ||
tok.IssuedBy != nil ||
tok.IssuedAt != nil {
@@ -1892,12 +1856,12 @@ func handleClientMessage(c *webClient, m clientMessage) error {
Value: tokens,
})
default:
return hall.UserError("unknown group action")
return hall.UserError("unknown hall action")
}
case "useraction":
g := c.hall
if g == nil {
return c.error(hall.UserError("join a group first"))
return c.error(hall.UserError("join a hall first"))
}
switch m.Kind {
case "op", "unop", "present", "unpresent", "shutup", "unshutup":
@@ -2076,7 +2040,7 @@ func parseStatefulToken(value interface{}) (*token.Stateful, error) {
if err != nil {
return nil, err
}
g, err := parseString("group")
g, err := parseString("hall")
if err != nil {
return nil, err
}
@@ -2098,7 +2062,7 @@ func parseStatefulToken(value interface{}) (*token.Stateful, error) {
}
return &token.Stateful{
Token: tt,
Group: gg,
Hall: gg,
Username: u,
Permissions: p,
Expires: e,
+3 -3
View File
@@ -11,18 +11,18 @@ import (
var tokens = []string{
`{
"token": "a",
"group": "g",
"hall": "h",
"username": "u",
"permissions":["present"],
"expires": "2023-05-03T20:24:47.616624532+02:00"
}`,
`{
"token": "a",
"group": "g"
"hall": "h"
}`,
`{
"token": "a",
"group": "g",
"hall": "h",
"username":""
}`,
}
+2 -2
View File
@@ -107,11 +107,11 @@ func (c *WhipClient) RequestConns(target hall.Client, g *hall.Hall, id string) e
return nil
}
func (c *WhipClient) Joined(group, kind string) error {
func (c *WhipClient) Joined(hall, kind string) error {
return nil
}
func (c *WhipClient) PushClient(group, kind, id, username string, permissions []string, status map[string]interface{}) error {
func (c *WhipClient) PushClient(hall, kind, id, username string, permissions []string, status map[string]interface{}) error {
return nil
}