Intro
When participating in an online meeting such as Microsoft Teams or Zoom, I can choose to share or not share video.
This time, I will try implementing it on my application.
Examples
Modifying a MediaStream
To change media streams during a session, it must be negotiated by sending offers and answers as at initiating connection.
Since it is the answer-side(client-side) that wants to change the media streams, I added a function to send messages to the offer-side(server-side) to request updating the session.
sseHub.go
...
func handleReceivedMessage(h *SSEHub, message ClientMessage) {
switch message.Event {
case TextEvent:
...
case CandidateEvent:
...
case AnswerEvent:
...
case UpdateEvent:
// when the offer-side is received this type messages,
// it will start updating the peer connections.
signalPeerConnections(h)
}
}
func signalPeerConnections(h *SSEHub) {
defer func() {
dispatchKeyFrame(h)
}()
for syncAttempt := 0; ; syncAttempt++ {
if syncAttempt == 25 {
// Release the lock and attempt a sync in 3 seconds. We might be blocking a RemoveTrack or AddTrack
go func() {
time.Sleep(time.Second * 3)
signalPeerConnections(h)
}()
return
}
if !attemptSync(h) {
break
}
}
}
...
Enabling / disabling video tracks
If the client-side application does not share any video during the session, the "getUserMedia" constraint can disable the use of video.
webrtc.controller.ts
...
navigator.mediaDevices.getUserMedia({ video: false, audio: true })
.then(stream => {
this.webcamStream = stream;
});
...
Adding a video track the first time
If I want to set the video enabled, I can execute "getUserMedia" again and add a video track into the MediaStream.
webrtc.controller.ts
...
private addVideoTrack(peerConnection: RTCPeerConnection) {
navigator.mediaDevices.getUserMedia({ video: true })
.then(stream => {
const newVideoTracks = stream.getVideoTracks();
if (this.webcamStream == null ||
newVideoTracks.length <= 0) {
return;
}
this.localVideo.srcObject = stream;
this.localVideo.play();
for (const v of newVideoTracks) {
this.webcamStream.addTrack(v);
peerConnection.addTrack(v, this.webcamStream);
}
if (this.connectionUpdatedEvent != null) {
this.connectionUpdatedEvent();
}
});
}
...
Removing the video and re-adding the video
I can stop sharing the video by "removeTrack".
However, if the local MediaStream video track is stopped, it will not be shared as a remote video track when it is re-added, so the local MediaStream is only paused.
webrtc.controller.ts
...
public switchLocalVideoUsage(used: boolean): void {
if (this.peerConnection == null ||
this.webcamStream == null) {
return;
}
const tracks = this.webcamStream.getVideoTracks();
if (used) {
if (tracks.length > 0 &&
tracks[0] != null) {
this.replaceVideoTrack(this.peerConnection, tracks[0]);
} else {
this.addVideoTrack(this.peerConnection);
}
} else {
this.removeVideoTrack(this.peerConnection);
}
}
...
/** for re-adding the video */
private replaceVideoTrack(peerConnection: RTCPeerConnection, track: MediaStreamTrack) {
this.localVideo.play();
for (const s of peerConnection.getSenders()) {
if (s.track == null || s.track.kind === "video") {
s.replaceTrack(track);
}
}
for (const t of peerConnection.getTransceivers()) {
if (t.sender.track?.kind == null ||
t.sender.track.kind === "video") {
t.direction = "sendrecv";
}
}
if (this.connectionUpdatedEvent != null) {
this.connectionUpdatedEvent();
}
}
private removeVideoTrack(peerConnection: RTCPeerConnection) {
const senders = peerConnection.getSenders();
if (senders.length > 0) {
this.localVideo.pause();
for (const s of senders) {
if (s.track?.kind === "video") {
peerConnection.removeTrack(s);
}
}
if (this.connectionUpdatedEvent != null) {
this.connectionUpdatedEvent();
}
}
}
...
Note that even after re-adding, the Transceiver direction will not be changed from "recvonly" and will be treated as "inactive" in the Answer's SDP, so it must be changed individually.
Adding / removing MediaStreams into the DOM
Previously, only MediaStreamTracks whose "kind" was "video" in all received were added as DOM elements.
This time, if there are no video tracks, audio tracks must be added as the audio elements.
Also, if the track is removed, the corresponding element must be deleted, but if only the video track is removed, the audio track must be taken from the "srcObject" of the video element and re-added as an audio element.
webrtc.controller.ts
import * as urlParam from "./urlParamGetter";
type RemoteTrack = {
id: string,
kind: "video"|"audio",
element: HTMLElement,
};
export class MainView {
...
public addRemoteTrack(stream: MediaStream, kind: "video"|"audio", id?: string): void {
if(this.tracks.some(t => t.id === stream.id)) {
if(kind === "audio") {
return;
}
this.removeRemoteTrack(stream.id, "audio");
}
const remoteTrack = document.createElement(kind);
remoteTrack.srcObject = stream;
remoteTrack.autoplay = true;
remoteTrack.controls = false;
this.remoteTrackArea.appendChild(remoteTrack);
this.tracks.push({
id: (id == null)? stream.id: id,
kind,
element: remoteTrack,
});
}
public removeRemoteTrack(id: string, kind: "video"|"audio"): void {
const targets = this.tracks.filter(t => t.id === id);
if(targets.length <= 0) {
return;
}
if(kind === "video") {
// the audio tracks must be re-added as audio elements.
const audioTrack = this.getAudioTrack(targets[0]?.element);
if(audioTrack != null) {
this.addRemoteTrack(new MediaStream([audioTrack]), "audio", id);
}
}
for(const t of targets) {
this.remoteTrackArea.removeChild(t.element);
}
const newTracks = new Array<RemoteTrack>();
for(const t of this.tracks.filter(t => t.id !== id || (t.id === id && t.kind !== kind))) {
newTracks.push(t);
}
this.tracks = newTracks;
}
/** get audio track from "srcObject" of HTMLVideoElements */
private getAudioTrack(target: HTMLElement|null|undefined): MediaStreamTrack|null {
if(target == null ||
!(target instanceof HTMLVideoElement)){
return null;
}
if(target.srcObject == null ||
!("getAudioTracks" in target.srcObject) ||
(typeof target.srcObject.getAudioTracks !== "function")) {
return null;
}
const tracks = target.srcObject.getAudioTracks();
if(tracks.length <= 0 ||
tracks[0] == null) {
return null;
}
return tracks[0];
}
}
Top comments (0)