From ec99373ae3422b925109602638f93e5bf35ca8ac Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Apr 2026 15:58:03 +0000 Subject: [PATCH 1/2] fix: show voice call UI when remote peer stops sharing video When a peer disables their camera, the remote side was showing a frozen last frame because the VideoTrack reference remained non-null. This adds a VideoSink-based frame monitor that detects when frames stop arriving and transitions the UI back to the voice call avatar view. https://claude.ai/code/session_01WdmqktFBjXKB6PYRZ5b4FD --- .../amethyst/service/call/CallController.kt | 44 ++++++++++++++++++- .../amethyst/ui/call/CallScreen.kt | 23 +++++----- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/CallController.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/CallController.kt index 0b761a6d4..88d57deb0 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/CallController.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/CallController.kt @@ -32,6 +32,7 @@ import com.vitorpamplona.quartz.nipACWebRtcCalls.tags.CallType import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -39,9 +40,12 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.webrtc.IceCandidate import org.webrtc.SessionDescription +import org.webrtc.VideoFrame +import org.webrtc.VideoSink import org.webrtc.VideoTrack import java.util.UUID import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicLong private const val TAG = "CallController" @@ -70,6 +74,16 @@ class CallController( private val _errorMessage = MutableStateFlow(null) val errorMessage: StateFlow = _errorMessage.asStateFlow() + // Remote video frame monitoring — detects when peer stops sending video + private val _isRemoteVideoActive = MutableStateFlow(false) + val isRemoteVideoActive: StateFlow = _isRemoteVideoActive.asStateFlow() + private val lastRemoteFrameTimeMs = AtomicLong(0L) + private var remoteVideoMonitorJob: kotlinx.coroutines.Job? = null + private val remoteFrameSink = + VideoSink { _: VideoFrame -> + lastRemoteFrameTimeMs.set(System.currentTimeMillis()) + } + // Audio/video toggle state (UI concerns, not domain state) private val _isAudioMuted = MutableStateFlow(false) val isAudioMuted: StateFlow = _isAudioMuted.asStateFlow() @@ -272,16 +286,41 @@ class CallController( audioManager.release() stopForegroundService() NotificationUtils.cancelCallNotification(context) + stopRemoteVideoMonitor() _remoteVideoTrack.value = null _localVideoTrack.value = null _isAudioMuted.value = false _isVideoEnabled.value = false + _isRemoteVideoActive.value = false webRtcSession?.dispose() webRtcSession = null remoteDescriptionSet.set(false) pendingIceCandidates.clear() } + private fun startRemoteVideoMonitor(track: VideoTrack) { + stopRemoteVideoMonitor() + lastRemoteFrameTimeMs.set(System.currentTimeMillis()) + track.addSink(remoteFrameSink) + remoteVideoMonitorJob = + scope.launch { + while (true) { + delay(1500) + val elapsed = System.currentTimeMillis() - lastRemoteFrameTimeMs.get() + _isRemoteVideoActive.value = elapsed < 2000 + } + } + } + + private fun stopRemoteVideoMonitor() { + remoteVideoMonitorJob?.cancel() + remoteVideoMonitorJob = null + try { + _remoteVideoTrack.value?.removeSink(remoteFrameSink) + } catch (_: Exception) { + } + } + private fun createWebRtcSession() { val session = WebRtcCallSession( @@ -292,7 +331,10 @@ class CallController( callManager.onPeerConnected() startForegroundService() }, - onRemoteVideoTrack = { track -> _remoteVideoTrack.value = track }, + onRemoteVideoTrack = { track -> + _remoteVideoTrack.value = track + startRemoteVideoMonitor(track) + }, onDisconnected = { scope.launch { callManager.hangup() } }, onError = { error -> _errorMessage.value = error }, ) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/call/CallScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/call/CallScreen.kt index 1f47e69bc..30ed7e3b6 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/call/CallScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/call/CallScreen.kt @@ -353,6 +353,7 @@ private fun ConnectedCallUI( val localVideoTrack by (callController?.localVideoTrack ?: emptyVideoFlow).collectAsState() val defaultFalse = remember { kotlinx.coroutines.flow.MutableStateFlow(false) } val defaultTrue = remember { kotlinx.coroutines.flow.MutableStateFlow(true) } + val isRemoteVideoActive by (callController?.isRemoteVideoActive ?: defaultFalse).collectAsState() val defaultRoute = remember { kotlinx.coroutines.flow.MutableStateFlow(AudioRoute.EARPIECE) } val isAudioMuted by (callController?.isAudioMuted ?: defaultFalse).collectAsState() val isVideoEnabled by (callController?.isVideoEnabled ?: defaultTrue).collectAsState() @@ -364,14 +365,16 @@ private fun ConnectedCallUI( .fillMaxSize() .background(Color.Black), ) { - // Remote video (full screen background) - remoteVideoTrack?.let { track -> - VideoRenderer( - videoTrack = track, - eglBase = callController?.getEglBase(), - modifier = Modifier.fillMaxSize(), - mirror = false, - ) + // Remote video (full screen background) — only when peer is actively sending + if (isRemoteVideoActive) { + remoteVideoTrack?.let { track -> + VideoRenderer( + videoTrack = track, + eglBase = callController?.getEglBase(), + modifier = Modifier.fillMaxSize(), + mirror = false, + ) + } } // Local video (small pip in corner) — only when camera is active @@ -390,8 +393,8 @@ private fun ConnectedCallUI( } } - // If no video, show avatar - if (remoteVideoTrack == null) { + // If no video or peer stopped sharing, show avatar + if (!isRemoteVideoActive) { Column( modifier = Modifier.fillMaxSize(), horizontalAlignment = Alignment.CenterHorizontally, From fe1a009fb4846f0c89c0c87e9c0f522b052c1301 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Apr 2026 17:05:10 +0000 Subject: [PATCH 2/2] fix: wire SDP renegotiation so video toggle works for remote peer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When toggling video mid-call, addVideoTrack triggers onRenegotiationNeeded but the callback was empty — the remote peer never received the updated SDP, so video never appeared on their side. This wires up the full renegotiation flow using the existing CallRenegotiateEvent (kind 25055): - WebRtcCallSession: forward onRenegotiationNeeded to a callback - CallController: create and send renegotiation offers, handle incoming renegotiation offers by creating answers - CallManager: route CallRenegotiateEvent, allow CallAnswerEvent during Connected state for renegotiation answers https://claude.ai/code/session_01WdmqktFBjXKB6PYRZ5b4FD --- .../amethyst/service/call/CallController.kt | 32 +++++++++++ .../service/call/WebRtcCallSession.kt | 6 ++- .../amethyst/commons/call/CallManager.kt | 54 +++++++++++++++++-- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/CallController.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/CallController.kt index 88d57deb0..cbdd6f3d9 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/CallController.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/CallController.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.amethyst.service.notifications.NotificationUtils import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.nipACWebRtcCalls.WebRtcCallFactory import com.vitorpamplona.quartz.nipACWebRtcCalls.events.CallIceCandidateEvent +import com.vitorpamplona.quartz.nipACWebRtcCalls.events.CallRenegotiateEvent import com.vitorpamplona.quartz.nipACWebRtcCalls.tags.CallType import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CoroutineScope @@ -93,6 +94,8 @@ class CallController( val isBluetoothAvailable: StateFlow = audioManager.isBluetoothAvailable init { + callManager.onRenegotiationOfferReceived = { event -> onRenegotiationOfferReceived(event) } + scope.launch { callManager.state.collect { state -> when (state) { @@ -282,6 +285,34 @@ class CallController( _errorMessage.value = null } + private fun onRenegotiationOfferReceived(event: CallRenegotiateEvent) { + val session = webRtcSession ?: return + val sdpOffer = event.sdpOffer() + Log.d(TAG) { "Renegotiation offer received, SDP length=${sdpOffer.length}" } + + scope.launch { + session.setRemoteDescription(SessionDescription(SessionDescription.Type.OFFER, sdpOffer)) + session.createAnswer { sdp -> + scope.launch { + callManager.sendRenegotiationAnswer(sdp.description) + } + } + } + } + + private fun performRenegotiation() { + val session = webRtcSession ?: return + val state = callManager.state.value + if (state !is CallState.Connected && state !is CallState.Connecting) return + + Log.d(TAG) { "Starting renegotiation" } + session.createOffer { sdp -> + scope.launch { + callManager.sendRenegotiation(sdp.description) + } + } + } + fun cleanup() { audioManager.release() stopForegroundService() @@ -337,6 +368,7 @@ class CallController( }, onDisconnected = { scope.launch { callManager.hangup() } }, onError = { error -> _errorMessage.value = error }, + onRenegotiationNeeded = { performRenegotiation() }, ) try { session.initialize() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/WebRtcCallSession.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/WebRtcCallSession.kt index 9981ec3c1..274b39399 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/WebRtcCallSession.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/call/WebRtcCallSession.kt @@ -52,6 +52,7 @@ class WebRtcCallSession( private val onRemoteVideoTrack: (VideoTrack) -> Unit, private val onDisconnected: () -> Unit, private val onError: (String) -> Unit = {}, + private val onRenegotiationNeeded: () -> Unit = {}, ) { private var peerConnectionFactory: PeerConnectionFactory? = null private var peerConnection: PeerConnection? = null @@ -131,7 +132,10 @@ class WebRtcCallSession( override fun onDataChannel(channel: DataChannel?) {} - override fun onRenegotiationNeeded() {} + override fun onRenegotiationNeeded() { + Log.d(TAG) { "Renegotiation needed" } + this@WebRtcCallSession.onRenegotiationNeeded() + } override fun onAddTrack( receiver: RtpReceiver?, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/call/CallManager.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/call/CallManager.kt index de2cddd8d..581969506 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/call/CallManager.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/call/CallManager.kt @@ -30,6 +30,7 @@ import com.vitorpamplona.quartz.nipACWebRtcCalls.events.CallHangupEvent import com.vitorpamplona.quartz.nipACWebRtcCalls.events.CallIceCandidateEvent import com.vitorpamplona.quartz.nipACWebRtcCalls.events.CallOfferEvent import com.vitorpamplona.quartz.nipACWebRtcCalls.events.CallRejectEvent +import com.vitorpamplona.quartz.nipACWebRtcCalls.events.CallRenegotiateEvent import com.vitorpamplona.quartz.nipACWebRtcCalls.tags.CallType import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quartz.utils.TimeUtils @@ -54,6 +55,7 @@ class CallManager( var onAnswerReceived: ((CallAnswerEvent) -> Unit)? = null var onIceCandidateReceived: ((CallIceCandidateEvent) -> Unit)? = null + var onRenegotiationOfferReceived: ((CallRenegotiateEvent) -> Unit)? = null private var timeoutJob: Job? = null private var resetJob: Job? = null @@ -119,12 +121,26 @@ class CallManager( fun onCallAnswered(event: CallAnswerEvent) { val current = _state.value - if (current !is CallState.Offering) return - if (event.callId() != current.callId) return + val callId = event.callId() - _state.value = CallState.Connecting(current.callId, current.peerPubKey, current.callType) - cancelTimeout() - onAnswerReceived?.invoke(event) + when (current) { + is CallState.Offering -> { + if (callId != current.callId) return + _state.value = CallState.Connecting(current.callId, current.peerPubKey, current.callType) + cancelTimeout() + onAnswerReceived?.invoke(event) + } + + is CallState.Connected -> { + // Renegotiation answer (e.g., peer accepted our video upgrade offer) + if (callId != current.callId) return + onAnswerReceived?.invoke(event) + } + + else -> { + return + } + } } fun onCallRejected(event: CallRejectEvent) { @@ -139,6 +155,33 @@ class CallManager( onIceCandidateReceived?.invoke(event) } + fun onRenegotiate(event: CallRenegotiateEvent) { + val current = _state.value + val callId = event.callId() + val currentCallId = + when (current) { + is CallState.Connecting -> current.callId + is CallState.Connected -> current.callId + else -> return + } + if (callId != currentCallId) return + onRenegotiationOfferReceived?.invoke(event) + } + + suspend fun sendRenegotiation(sdpOffer: String) { + val callId = currentCallId() ?: return + val peerPubKey = currentPeerPubKey() ?: return + val result = factory.createRenegotiate(sdpOffer, peerPubKey, callId, signer) + publishEvent(result.wrap) + } + + suspend fun sendRenegotiationAnswer(sdpAnswer: String) { + val callId = currentCallId() ?: return + val peerPubKey = currentPeerPubKey() ?: return + val result = factory.createCallAnswer(sdpAnswer, peerPubKey, callId, signer) + publishEvent(result.wrap) + } + fun onPeerConnected() { val current = _state.value if (current !is CallState.Connecting) return @@ -213,6 +256,7 @@ class CallManager( is CallRejectEvent -> onCallRejected(event) is CallHangupEvent -> onPeerHangup(event) is CallIceCandidateEvent -> onIceCandidate(event) + is CallRenegotiateEvent -> onRenegotiate(event) } }