Merge pull request #2115 from vitorpamplona/claude/fix-webrtc-multiple-videos-EXBGs

Support multiple remote video tracks in group calls
2026-04-03 16:10:58 -04:00
parent 4cf9418582 6bb464e768
commit e36a6e4ae2
2 changed files with 111 additions and 11 deletions
@@ -138,6 +138,10 @@ class CallController(
            }
        }

+    // Per-peer video activity monitoring for group calls
+    private val perPeerFrameSinks = ConcurrentHashMap<HexKey, VideoSink>()
+    private val perPeerLastFrameTimeMs = ConcurrentHashMap<HexKey, AtomicLong>()
+
    private val _isAudioMuted = MutableStateFlow(false)
    val isAudioMuted: StateFlow<Boolean> = _isAudioMuted.asStateFlow()
    private val _isVideoEnabled = MutableStateFlow(false)
@@ -725,6 +729,49 @@ class CallController(
            _remoteVideoTrack.value = track
            startRemoteVideoMonitor(track)
        }
+        // Monitor this peer's video activity for group call UI
+        startPeerVideoMonitor(peerPubKey, track)
+    }
+
+    private fun startPeerVideoMonitor(
+        peerPubKey: HexKey,
+        track: VideoTrack,
+    ) {
+        // Remove any existing sink for this peer
+        stopPeerVideoMonitor(peerPubKey)
+
+        val lastFrameTime = AtomicLong(System.currentTimeMillis())
+        perPeerLastFrameTimeMs[peerPubKey] = lastFrameTime
+        val sink =
+            VideoSink { frame: VideoFrame ->
+                lastFrameTime.set(System.currentTimeMillis())
+            }
+        perPeerFrameSinks[peerPubKey] = sink
+        track.addSink(sink)
+        ensureGroupVideoMonitorRunning()
+    }
+
+    private fun stopPeerVideoMonitor(peerPubKey: HexKey) {
+        val sink = perPeerFrameSinks.remove(peerPubKey) ?: return
+        perPeerLastFrameTimeMs.remove(peerPubKey)
+        val track = _remoteVideoTracks.value[peerPubKey]
+        try {
+            track?.removeSink(sink)
+        } catch (_: Exception) {
+        }
+    }
+
+    private fun ensureGroupVideoMonitorRunning() {
+        if (remoteVideoMonitorJob != null) return
+        remoteVideoMonitorJob =
+            scope.launch {
+                while (true) {
+                    delay(1500)
+                    val now = System.currentTimeMillis()
+                    val anyActive = perPeerLastFrameTimeMs.values.any { now - it.get() < 2000 }
+                    _isRemoteVideoActive.value = anyActive || (now - lastRemoteFrameTimeMs.get() < 2000)
+                }
+            }
    }

    private fun onPeerDisconnected(peerPubKey: HexKey) {
@@ -764,6 +811,8 @@ class CallController(
            } catch (e: Exception) {
                Log.e(TAG, "disposePeerSession: dispose() failed for ${peerPubKey.take(8)}", e)
            }
+            // Clean up per-peer video monitor
+            stopPeerVideoMonitor(peerPubKey)
            // Update remote video tracks
            val currentTracks = _remoteVideoTracks.value
            if (peerPubKey in currentTracks) {
@@ -807,6 +856,11 @@ class CallController(
        NotificationUtils.cancelCallNotification(context)
        stopRemoteVideoMonitor()

+        // Clean up per-peer video monitors
+        for (peerPubKey in perPeerFrameSinks.keys.toList()) {
+            stopPeerVideoMonitor(peerPubKey)
+        }
+
        // Dispose all peer sessions
        for (ps in peerSessions.values) {
            try {
@@ -29,6 +29,7 @@ import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.Row
 import androidx.compose.foundation.layout.Spacer
 import androidx.compose.foundation.layout.WindowInsets
+import androidx.compose.foundation.layout.fillMaxHeight
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.foundation.layout.height
@@ -399,7 +400,8 @@ private fun ConnectedCallUI(
    }

    val emptyVideoFlow = remember { kotlinx.coroutines.flow.MutableStateFlow<VideoTrack?>(null) }
-    val remoteVideoTrack by (callController?.remoteVideoTrack ?: emptyVideoFlow).collectAsState()
+    val emptyTracksFlow = remember { kotlinx.coroutines.flow.MutableStateFlow<Map<String, VideoTrack>>(emptyMap()) }
+    val remoteVideoTracks by (callController?.remoteVideoTracks ?: emptyTracksFlow).collectAsState()
    val localVideoTrack by (callController?.localVideoTrack ?: emptyVideoFlow).collectAsState()
    val defaultFalse = remember { kotlinx.coroutines.flow.MutableStateFlow(false) }
    val defaultTrue = remember { kotlinx.coroutines.flow.MutableStateFlow(true) }
@@ -429,16 +431,13 @@ private fun ConnectedCallUI(
                .fillMaxSize()
                .background(Color.Black),
    ) {
-        // Remote video (full screen background) — only when peer is actively sending
-        if (isRemoteVideoActive) {
-            remoteVideoTrack?.let { track ->
-                VideoRenderer(
-                    videoTrack = track,
-                    eglBase = callController?.getEglBase(),
-                    modifier = Modifier.fillMaxSize(),
-                    mirror = false,
-                )
-            }
+        // Remote video(s) — render all peers in a grid when actively sending
+        if (isRemoteVideoActive && remoteVideoTracks.isNotEmpty()) {
+            RemoteVideoGrid(
+                remoteVideoTracks = remoteVideoTracks,
+                eglBase = callController?.getEglBase(),
+                modifier = Modifier.fillMaxSize(),
+            )
        }

        // Local video (small pip in corner) — only when camera is active
@@ -602,6 +601,53 @@ private fun ConnectedCallUI(
    }
 }

+@Composable
+private fun RemoteVideoGrid(
+    remoteVideoTracks: Map<String, VideoTrack>,
+    eglBase: org.webrtc.EglBase?,
+    modifier: Modifier = Modifier,
+) {
+    val tracks = remember(remoteVideoTracks) { remoteVideoTracks.entries.toList() }
+
+    if (tracks.size == 1) {
+        // Single peer: full screen
+        VideoRenderer(
+            videoTrack = tracks[0].value,
+            eglBase = eglBase,
+            modifier = modifier,
+            mirror = false,
+        )
+    } else {
+        val columns =
+            when {
+                tracks.size <= 2 -> 1
+                tracks.size <= 4 -> 2
+                else -> 2
+            }
+
+        Column(modifier = modifier) {
+            tracks.chunked(columns).forEach { row ->
+                Row(
+                    modifier = Modifier.weight(1f).fillMaxWidth(),
+                ) {
+                    row.forEach { (_, track) ->
+                        VideoRenderer(
+                            videoTrack = track,
+                            eglBase = eglBase,
+                            modifier = Modifier.weight(1f).fillMaxHeight(),
+                            mirror = false,
+                        )
+                    }
+                    // Fill empty cells in the last row
+                    repeat(columns - row.size) {
+                        Spacer(modifier = Modifier.weight(1f))
+                    }
+                }
+            }
+        }
+    }
+}
+
@Composable
 private fun VideoRenderer(
    videoTrack: VideoTrack,