storm/skald
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Label streams, not tracks.

Browse Source 
We used to label tracks individually, in a view to using the labelling
for simulcast.  Since then, the WebRTC community has converged on a
different strategy, where multiple tracks share a single mid and
are labelled with the rid extension.

We now label whole streams, which is simpler, and use the track's
kind (and, in the future, the rid) to disambiguate.  This changes the
protocol in two ways:

  * in offers, the "labels" dictionary is replaced by a single "label"
    field; and
  * the syntax of the "request" message has changed.
This commit is contained in:
Juliusz Chroboczek
2021-04-29 18:28:16 +02:00
parent b08a2e3943
commit be73380f9f
7 changed files with 147 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/galene.js
+56 -36
View File
@@ -342,13 +342,7 @@ function gotDownStream(c) {
setMedia(c, false);
};
c.onnegotiationcompleted = function() {
let found = false;
for(let key in c.labels) {
if(c.labels[key] === 'video')
found = true;
}
if(!found)
resetMedia(c);
resetMedia(c);
}
c.onstatus = function(status) {
setMediaStatus(c);
@@ -384,7 +378,7 @@ getButtonElement('presentbutton').onclick = async function(e) {
// button a second time before the stream is set up and the button hidden.
button.disabled = true;
try {
let id = findUpMedia('local');
let id = findUpMedia('camera');
if(!id)
await addLocalMedia();
} finally {
@@ -394,12 +388,12 @@ getButtonElement('presentbutton').onclick = async function(e) {
getButtonElement('unpresentbutton').onclick = function(e) {
e.preventDefault();
closeUpMediaKind('local');
closeUpMedia('camera');
resizePeers();
};
function changePresentation() {
let c = findUpMedia('local');
let c = findUpMedia('camera');
if(c)
addLocalMedia(c.localId);
}
@@ -419,7 +413,7 @@ function setVisibility(id, visible) {
function setButtonsVisibility() {
let connected = serverConnection && serverConnection.socket;
let permissions = serverConnection.permissions;
let local = !!findUpMedia('local');
let local = !!findUpMedia('camera');
let video = !!findUpMedia('video');
let canWebrtc = !(typeof RTCPeerConnection === 'undefined');
let canFile =
@@ -516,7 +510,7 @@ document.getElementById('sharebutton').onclick = function(e) {
document.getElementById('stopvideobutton').onclick = function(e) {
e.preventDefault();
closeUpMediaKind('video');
closeUpMedia('video');
resizePeers();
};
@@ -556,12 +550,37 @@ getSelectElement('sendselect').onchange = async function(e) {
}
};
/**
* @param {string} what
* @returns {Object<string,Array<string>>}
*/
function mapRequest(what) {
switch(what) {
case '':
return {};
break;
case 'audio':
return {'': ['audio']};
break;
case 'screenshare':
return {screenshare: ['audio','video'], '': ['audio']};
break;
case 'everything':
return {'': ['audio','video']}
break;
default:
displayError(`Unknown value ${what} in request`);
return {};
}
}
getSelectElement('requestselect').onchange = function(e) {
e.preventDefault();
if(!(this instanceof HTMLSelectElement))
throw new Error('Unexpected type for this');
updateSettings({request: this.value});
serverConnection.request(this.value);
serverConnection.request(mapRequest(this.value));
};
const activityDetectionInterval = 200;
@@ -1069,8 +1088,8 @@ async function addLocalMedia(localId) {
return;
}
c.kind = 'local';
c.stream = stream;
c.label = 'camera';
if(filter) {
try {
@@ -1100,7 +1119,6 @@ async function addLocalMedia(localId) {
let mute = getSettings().localMute;
c.stream.getTracks().forEach(t => {
c.labels[t.id] = t.kind;
if(t.kind == 'audio') {
if(mute)
t.enabled = false;
@@ -1143,8 +1161,8 @@ async function addShareMedia() {
}
let c = newUpStream();
c.kind = 'screenshare';
c.stream = stream;
c.label = 'screenshare';
c.onclose = replace => {
stopStream(stream);
if(!replace)
@@ -1153,7 +1171,6 @@ async function addShareMedia() {
stream.getTracks().forEach(t => {
c.pc.addTrack(t, stream);
t.onended = e => c.close();
c.labels[t.id] = 'screenshare';
});
c.onstats = gotUpStats;
c.setStatsInterval(2000);
@@ -1184,8 +1201,8 @@ async function addFileMedia(file) {
}
let c = newUpStream();
c.kind = 'video';
c.stream = stream;
c.label = 'video';
c.onclose = function(replace) {
stopStream(c.stream);
let media = /** @type{HTMLVideoElement} */
@@ -1201,7 +1218,7 @@ async function addFileMedia(file) {
stream.onaddtrack = function(e) {
let t = e.track;
if(t.kind === 'audio') {
let presenting = !!findUpMedia('local');
let presenting = !!findUpMedia('camera');
let muted = getSettings().localMute;
if(presenting && !muted) {
setLocalMute(true, true);
@@ -1209,13 +1226,11 @@ async function addFileMedia(file) {
}
}
c.pc.addTrack(t, stream);
c.labels[t.id] = t.kind;
c.onstats = gotUpStats;
c.setStatsInterval(2000);
};
stream.onremovetrack = function(e) {
let t = e.track;
delete(c.labels[t.id]);
/** @type {RTCRtpSender} */
let sender;
@@ -1229,7 +1244,12 @@ async function addFileMedia(file) {
console.warn('Removing unknown track');
}
if(Object.keys(c.labels).length === 0) {
let found = false;
c.pc.getSenders().forEach(s => {
if(s.track)
found = true;
});
if(!found) {
stream.onaddtrack = null;
stream.onremovetrack = null;
c.close();
@@ -1254,28 +1274,28 @@ function stopStream(s) {
}
/**
* closeUpMediaKind closes all up connections that correspond to a given
* kind of media. If kind is null, it closes all up connections.
* closeUpMedia closes all up connections with the given label. If label
* is null, it closes all up connections.
*
* @param {string} kind
* @param {string} label
*/
function closeUpMediaKind(kind) {
function closeUpMedia(label) {
for(let id in serverConnection.up) {
let c = serverConnection.up[id];
if(kind && c.kind != kind)
if(label && c.label !== label)
continue
c.close();
}
}
/**
* @param {string} kind
* @param {string} label
* @returns {Stream}
*/
function findUpMedia(kind) {
function findUpMedia(label) {
for(let id in serverConnection.up) {
let c = serverConnection.up[id]
if(c.kind === kind)
if(c.label === label)
return c;
}
return null;
@@ -1289,7 +1309,7 @@ function muteLocalTracks(mute) {
return;
for(let id in serverConnection.up) {
let c = serverConnection.up[id];
if(c.kind === 'local') {
if(c.label === 'camera') {
let stream = c.stream;
stream.getTracks().forEach(t => {
if(t.kind === 'audio') {
@@ -1370,7 +1390,7 @@ async function setMedia(c, isUp, mirror, video) {
showVideo();
resizePeers();
if(!isUp && isSafari() && !findUpMedia('local')) {
if(!isUp && isSafari() && !findUpMedia('camera')) {
// Safari doesn't allow autoplay unless the user has granted media access
try {
let stream = await navigator.mediaDevices.getUserMedia({audio: true});
@@ -1429,10 +1449,10 @@ function addCustomControls(media, container, c) {
let volume = getVideoButton(controls, 'volume');
let stopsharing = getVideoButton(topcontrols, 'video-stop');
if (c.kind !== "screenshare") {
if (c.label !== "screenshare") {
stopsharing.remove();
}
if(c.kind === 'local') {
if(c.label === 'camera') {
volume.remove();
} else {
setVolumeButton(media.muted,
@@ -1820,9 +1840,9 @@ async function gotJoined(kind, group, perms, message) {
if(typeof RTCPeerConnection === 'undefined')
displayWarning("This browser doesn't support WebRTC");
else
this.request(getSettings().request);
this.request(mapRequest(getSettings().request));
if(serverConnection.permissions.present && !findUpMedia('local')) {
if(serverConnection.permissions.present && !findUpMedia('camera')) {
if(present) {
if(present === 'mike')
updateSettings({video: ''});
static/protocol.js
+16 -73
View File
@@ -212,8 +212,8 @@ function ServerConnection() {
* @property {boolean} [noecho]
* @property {string} [sdp]
* @property {RTCIceCandidate} [candidate]
* @property {Object<string,string>} [labels]
* @property {Object<string,(boolean|number)>} [request]
* @property {string} [label]
* @property {Object<string,Array<string>>} [request]
* @property {Object<string,any>} [rtcConfiguration]
*/
@@ -296,7 +296,7 @@ ServerConnection.prototype.connect = async function(url) {
case 'handshake':
break;
case 'offer':
sc.gotOffer(m.id, m.labels, m.source, m.username,
sc.gotOffer(m.id, m.label, m.source, m.username,
m.sdp, m.replace);
break;
case 'answer':
@@ -439,32 +439,14 @@ ServerConnection.prototype.leave = function(group) {
/**
* request sets the list of requested media types.
*
* @param {string} what - One of '', 'audio', 'screenshare' or 'everything'.
* @param {Object<string,Array<string>>} what
* - A dictionary that maps labels to a sequence of 'audio' and 'video'.
* An entry with an empty label '' provides the default.
*/
ServerConnection.prototype.request = function(what) {
/** @type {Object<string,boolean>} */
let request = {};
switch(what) {
case '':
request = {};
break;
case 'audio':
request = {audio: true};
break;
case 'screenshare':
request = {audio: true, screenshare: true};
break;
case 'everything':
request = {audio: true, screenshare: true, video: true};
break;
default:
console.error(`Unknown value ${what} in request`);
break;
}
this.send({
type: 'request',
request: request,
request: what,
});
};
@@ -622,14 +604,14 @@ ServerConnection.prototype.groupAction = function(kind, message) {
* Called when we receive an offer from the server. Don't call this.
*
* @param {string} id
* @param {Object<string, string>} labels
* @param {string} label
* @param {string} source
* @param {string} username
* @param {string} sdp
* @param {string} replace
* @function
*/
ServerConnection.prototype.gotOffer = async function(id, labels, source, username, sdp, replace) {
ServerConnection.prototype.gotOffer = async function(id, label, source, username, sdp, replace) {
let sc = this;
if(sc.up[id]) {
@@ -669,6 +651,7 @@ ServerConnection.prototype.gotOffer = async function(id, labels, source, usernam
return;
}
c = new Stream(this, id, oldLocalId || newLocalId(), pc, false);
c.label = label;
sc.down[id] = c;
c.pc.onicecandidate = function(e) {
@@ -689,32 +672,15 @@ ServerConnection.prototype.gotOffer = async function(id, labels, source, usernam
};
c.pc.ontrack = function(e) {
let label = e.transceiver && c.labelsByMid[e.transceiver.mid];
if(label) {
c.labels[e.track.id] = label;
} else {
console.warn("Couldn't find label for track");
}
c.stream = e.streams[0];
if(c.ondowntrack) {
c.ondowntrack.call(
c, e.track, e.transceiver, label, e.streams[0],
c, e.track, e.transceiver, e.streams[0],
);
}
};
}
c.labelsByMid = labels;
c.labels = {};
c.pc.getTransceivers().forEach(transceiver => {
let label = c.labelsByMid[transceiver.mid];
let track = transceiver.receiver && transceiver.receiver.track;
if(label && track) {
c.labels[track.id] = label;
} else if(!track) {
console.warn("Couldn't find track for label");
}
});
c.source = source;
c.username = username;
@@ -883,12 +849,6 @@ function Stream(sc, id, localId, pc, up) {
* @const
*/
this.up = up;
/**
* For up streams, one of "local" or "screenshare".
*
* @type {string}
*/
this.kind = null;
/**
* For down streams, the id of the client that created the stream.
*
@@ -916,17 +876,11 @@ function Stream(sc, id, localId, pc, up) {
*/
this.stream = null;
/**
* Track labels, indexed by track id.
* The label assigned by the originator to this stream.
*
* @type {Object<string,string>}
* @type {string}
*/
this.labels = {};
/**
* Track labels, indexed by mid.
*
* @type {Object<string,string>}
*/
this.labelsByMid = {};
this.label = null;
/**
* The id of the stream that we are currently replacing.
*
@@ -1004,7 +958,7 @@ function Stream(sc, id, localId, pc, up) {
* If the stream parameter differs from its previous value, then it
* indicates that the old stream has been discarded.
*
* @type{(this: Stream, track: MediaStreamTrack, transceiver: RTCRtpTransceiver, label: string, stream: MediaStream) => void}
* @type{(this: Stream, track: MediaStreamTrack, transceiver: RTCRtpTransceiver, stream: MediaStream) => void}
*/
this.ondowntrack = null;
/**
@@ -1164,17 +1118,6 @@ Stream.prototype.negotiate = async function (restartIce) {
throw(new Error("Didn't create offer"));
await c.pc.setLocalDescription(offer);
// mids are not known until this point
c.pc.getTransceivers().forEach(t => {
if(t.sender && t.sender.track) {
let label = c.labels[t.sender.track.id];
if(label)
c.labelsByMid[t.mid] = label;
else
console.warn("Couldn't find label for track");
}
});
c.sc.send({
type: 'offer',
source: c.sc.id,
@@ -1182,7 +1125,7 @@ Stream.prototype.negotiate = async function (restartIce) {
kind: this.localDescriptionSent ? 'renegotiate' : '',
id: c.id,
replace: this.replace,
labels: c.labelsByMid,
label: c.label,
sdp: c.pc.localDescription.sdp,
});
this.localDescriptionSent = true;