feat(nests): pulse the speaker ring with live audio level

Adds an end-to-end audio-amplitude pipeline so on-stage speakers'
green ring throbs in time with their voice (closer to Spaces /
Clubhouse than the previous binary "is in speakingNow" indicator).

NestPlayer.play() now takes an onLevel callback and computes the
normalized peak of each decoded 16-bit PCM frame. NestViewModel
exposes audioLevels: StateFlow<Map<String, Float>>; raw 50 Hz
updates from the decode loop are coalesced into a 10 Hz publish
tick (LEVEL_TICK_MS) so the StateFlow doesn't spam recompositions
across a busy stage. The map is cleared on speaker close, on the
speaking-timeout sweep, and on teardown.

In MemberCell the speaking ring's width animates between
AVATAR_RING_WIDTH (3 dp) and MAX_RING_WIDTH (7 dp) via
animateDpAsState, smoothing the tick into a continuous halo.
Muted-publisher and idle states are unchanged.

Adds NestPlayerTest coverage for the new callback (level math +
empty-PCM short-circuit).
This commit is contained in:
Claude
2026-04-28 12:03:13 +00:00
parent 154b3393a5
commit 04ee2c3106
5 changed files with 177 additions and 2 deletions
@@ -134,6 +134,7 @@ internal fun NestFullScreen(
val presences by viewModel.presences.collectAsState() val presences by viewModel.presences.collectAsState()
val reactionsByPubkey by viewModel.recentReactions.collectAsState() val reactionsByPubkey by viewModel.recentReactions.collectAsState()
val speakerCatalogs by viewModel.speakerCatalogs.collectAsState() val speakerCatalogs by viewModel.speakerCatalogs.collectAsState()
val audioLevels by viewModel.audioLevels.collectAsState()
val onStageKeys = remember(onStage) { onStage.map { it.pubKey }.toSet() } val onStageKeys = remember(onStage) { onStage.map { it.pubKey }.toSet() }
val participantGrid = val participantGrid =
@@ -225,6 +226,7 @@ internal fun NestFullScreen(
StageGrid( StageGrid(
members = participantGrid.onStage, members = participantGrid.onStage,
speakingNow = ui.speakingNow, speakingNow = ui.speakingNow,
audioLevels = audioLevels,
accountViewModel = accountViewModel, accountViewModel = accountViewModel,
reactionsByPubkey = reactionsByPubkey, reactionsByPubkey = reactionsByPubkey,
connectingSpeakers = ui.connectingSpeakers, connectingSpeakers = ui.connectingSpeakers,
@@ -22,6 +22,7 @@ package com.vitorpamplona.amethyst.ui.screen.loggedIn.nests.room.stage
import androidx.compose.animation.core.FastOutSlowInEasing import androidx.compose.animation.core.FastOutSlowInEasing
import androidx.compose.animation.core.RepeatMode import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateDpAsState
import androidx.compose.animation.core.animateFloat import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition import androidx.compose.animation.core.rememberInfiniteTransition
@@ -74,6 +75,11 @@ private val AUDIENCE_AVATAR = 100.dp
private val GRID_SPACING = 6.dp private val GRID_SPACING = 6.dp
private val AVATAR_RING_WIDTH = 3.dp private val AVATAR_RING_WIDTH = 3.dp
// Upper bound for the live "voice" halo. Speaking with a peak audio
// level of 1.0 grows the ring from AVATAR_RING_WIDTH to this value;
// quieter voices land somewhere in between.
private val MAX_RING_WIDTH = 7.dp
// Cap badge sizes so they stay legible without dominating the avatar // Cap badge sizes so they stay legible without dominating the avatar
// at 100.dp. The 0.42 ratio was tuned for ~48.dp avatars (giving // at 100.dp. The 0.42 ratio was tuned for ~48.dp avatars (giving
// ~20.dp badges); without a cap, scaling to 100.dp produces 42.dp // ~20.dp badges); without a cap, scaling to 100.dp produces 42.dp
@@ -109,6 +115,7 @@ internal fun StageGrid(
speakingNow: ImmutableSet<String>, speakingNow: ImmutableSet<String>,
accountViewModel: AccountViewModel, accountViewModel: AccountViewModel,
modifier: Modifier = Modifier, modifier: Modifier = Modifier,
audioLevels: Map<String, Float> = emptyMap(),
reactionsByPubkey: Map<String, List<RoomReaction>> = emptyMap(), reactionsByPubkey: Map<String, List<RoomReaction>> = emptyMap(),
connectingSpeakers: ImmutableSet<String> = persistentSetOf(), connectingSpeakers: ImmutableSet<String> = persistentSetOf(),
onLongPressParticipant: ((String) -> Unit)? = null, onLongPressParticipant: ((String) -> Unit)? = null,
@@ -141,6 +148,7 @@ internal fun StageGrid(
member = member, member = member,
avatarSize = STAGE_AVATAR, avatarSize = STAGE_AVATAR,
isSpeaking = member.pubkey in speakingNow, isSpeaking = member.pubkey in speakingNow,
audioLevel = audioLevels[member.pubkey] ?: 0f,
isConnecting = member.pubkey in connectingSpeakers, isConnecting = member.pubkey in connectingSpeakers,
showMicBadge = true, showMicBadge = true,
reactions = reactionsByPubkey[member.pubkey].orEmpty(), reactions = reactionsByPubkey[member.pubkey].orEmpty(),
@@ -205,6 +213,7 @@ internal fun AudienceGrid(
member = member, member = member,
avatarSize = AUDIENCE_AVATAR, avatarSize = AUDIENCE_AVATAR,
isSpeaking = false, isSpeaking = false,
audioLevel = 0f,
isConnecting = false, isConnecting = false,
showMicBadge = false, showMicBadge = false,
reactions = emptyList(), reactions = emptyList(),
@@ -221,6 +230,7 @@ private fun MemberCell(
member: RoomMember, member: RoomMember,
avatarSize: Dp, avatarSize: Dp,
isSpeaking: Boolean, isSpeaking: Boolean,
audioLevel: Float,
isConnecting: Boolean, isConnecting: Boolean,
showMicBadge: Boolean, showMicBadge: Boolean,
reactions: List<RoomReaction>, reactions: List<RoomReaction>,
@@ -239,9 +249,20 @@ private fun MemberCell(
showMicBadge && member.publishing && member.muted == true -> mutedRingColor showMicBadge && member.publishing && member.muted == true -> mutedRingColor
else -> null else -> null
} }
// Throb the ring width with the live peak amplitude — quiet voices
// sit at the base width, loud ones widen the halo. animateDpAsState
// smooths the 10 Hz raw signal into a continuous animation. The
// muted ring stays at base width because no audio is being decoded.
val targetRingWidth =
when {
isSpeaking -> AVATAR_RING_WIDTH + (audioLevel.coerceIn(0f, 1f) * (MAX_RING_WIDTH - AVATAR_RING_WIDTH).value).dp
ringColor != null -> AVATAR_RING_WIDTH
else -> 0.dp
}
val animatedRingWidth by animateDpAsState(targetValue = targetRingWidth, label = "speaker-ring-width")
val avatarModifier = val avatarModifier =
Modifier Modifier
.let { if (ringColor != null) it.border(AVATAR_RING_WIDTH, ringColor, CircleShape) else it } .let { if (ringColor != null) it.border(animatedRingWidth, ringColor, CircleShape) else it }
.let { if (member.absent) it.alpha(0.5f) else it } .let { if (member.absent) it.alpha(0.5f) else it }
val user = val user =
remember(member.pubkey) { remember(member.pubkey) {
@@ -189,6 +189,24 @@ class NestViewModel(
private val _announcedSpeakers = MutableStateFlow<Set<String>>(emptySet()) private val _announcedSpeakers = MutableStateFlow<Set<String>>(emptySet())
val announcedSpeakers: StateFlow<Set<String>> = _announcedSpeakers.asStateFlow() val announcedSpeakers: StateFlow<Set<String>> = _announcedSpeakers.asStateFlow()
/**
* Per-speaker peak audio amplitude for the most recent decoded
* frame, normalized to `[0, 1]`. Drives the live "voice ring"
* around speaker avatars: while a pubkey is in [NestUiState.speakingNow],
* the UI reads this map to throb the green border in time with
* the voice.
*
* Updated by [NestPlayer]'s `onLevel` callback at ~50 Hz per speaker
* (one frame per 20 ms Opus packet); the VM coalesces those raw
* updates into a single StateFlow emission every [LEVEL_TICK_MS]
* via [levelEmitterJob]. Empty when no speaker is being decoded;
* an entry drops when its subscription closes.
*/
private val rawAudioLevels = mutableMapOf<String, Float>()
private val _audioLevels = MutableStateFlow<Map<String, Float>>(emptyMap())
val audioLevels: StateFlow<Map<String, Float>> = _audioLevels.asStateFlow()
private var levelEmitterJob: Job? = null
/** /**
* `true` once the local user has been kicked (#5) — the platform * `true` once the local user has been kicked (#5) — the platform
* layer flips this on a valid kind-4312 from a host/moderator and * layer flips this on a valid kind-4312 from a host/moderator and
@@ -638,6 +656,7 @@ class NestViewModel(
listener = l listener = l
observeListenerState(l) observeListenerState(l)
observeAnnounces(l) observeAnnounces(l)
startLevelEmitter()
} catch (ce: CancellationException) { } catch (ce: CancellationException) {
throw ce throw ce
} catch (t: Throwable) { } catch (t: Throwable) {
@@ -695,6 +714,9 @@ class NestViewModel(
if (_speakerCatalogs.value.containsKey(slot.pubkey)) { if (_speakerCatalogs.value.containsKey(slot.pubkey)) {
_speakerCatalogs.update { it - slot.pubkey } _speakerCatalogs.update { it - slot.pubkey }
} }
if (rawAudioLevels.remove(slot.pubkey) != null) {
_audioLevels.value = rawAudioLevels.toMap()
}
if (roomPlayer != null || handle != null) { if (roomPlayer != null || handle != null) {
viewModelScope.launch { viewModelScope.launch {
roomPlayer?.runCatching { stop() } roomPlayer?.runCatching { stop() }
@@ -773,7 +795,11 @@ class NestViewModel(
// Tap the object flow to drive the speaking-now indicator before // Tap the object flow to drive the speaking-now indicator before
// the decoder consumes it. // the decoder consumes it.
val instrumented = handle.objects.onEach { onSpeakerActivity(pubkey) } val instrumented = handle.objects.onEach { onSpeakerActivity(pubkey) }
roomPlayer.play(instrumented, onError = { /* swallow per-packet decoder errors */ }) roomPlayer.play(
instrumented,
onError = { /* swallow per-packet decoder errors */ },
onLevel = { onAudioLevel(pubkey, it) },
)
slot.attach(handle, roomPlayer, player) slot.attach(handle, roomPlayer, player)
publishActiveSpeakers() publishActiveSpeakers()
// Enter the buffering window — UI renders a spinner // Enter the buffering window — UI renders a spinner
@@ -817,6 +843,12 @@ class NestViewModel(
stateObserverJob = null stateObserverJob = null
announcesJob?.cancel() announcesJob?.cancel()
announcesJob = null announcesJob = null
levelEmitterJob?.cancel()
levelEmitterJob = null
rawAudioLevels.clear()
if (_audioLevels.value.isNotEmpty()) {
_audioLevels.value = emptyMap()
}
if (_announcedSpeakers.value.isNotEmpty()) { if (_announcedSpeakers.value.isNotEmpty()) {
_announcedSpeakers.value = emptySet() _announcedSpeakers.value = emptySet()
} }
@@ -906,6 +938,48 @@ class NestViewModel(
if (_uiState.value.speakingNow.contains(pubkey)) { if (_uiState.value.speakingNow.contains(pubkey)) {
_uiState.update { it.copy(speakingNow = (it.speakingNow - pubkey).toPersistentSet()) } _uiState.update { it.copy(speakingNow = (it.speakingNow - pubkey).toPersistentSet()) }
} }
// Drop the latest level too — when the speaker goes quiet the
// ring should fall back to the static "in speakingNow" colour
// rather than freezing at the last loud peak.
if (rawAudioLevels.remove(pubkey) != null) {
_audioLevels.value = rawAudioLevels.toMap()
}
}
/**
* Record the latest decoded peak amplitude for [pubkey]. Called
* from the [NestPlayer] decode loop on the same dispatcher as the
* VM, so plain map mutation is safe. The actual StateFlow emission
* is coalesced by [startLevelEmitter] so a 50 Hz packet rate
* doesn't translate into 50 Hz recompositions.
*/
private fun onAudioLevel(
pubkey: String,
level: Float,
) {
if (closed) return
rawAudioLevels[pubkey] = level
}
/**
* Tick every [LEVEL_TICK_MS] and publish the current map of
* per-speaker levels. Coalesces the high-frequency raw updates
* into ~10 Hz UI state so the speaking-ring animation has a
* smooth, lightweight signal to follow.
*/
private fun startLevelEmitter() {
if (levelEmitterJob?.isActive == true) return
levelEmitterJob =
viewModelScope.launch {
while (true) {
delay(LEVEL_TICK_MS)
if (closed) return@launch
val snapshot = if (rawAudioLevels.isEmpty()) emptyMap() else rawAudioLevels.toMap()
if (snapshot != _audioLevels.value) {
_audioLevels.value = snapshot
}
}
}
} }
private class ActiveSubscription private constructor( private class ActiveSubscription private constructor(
@@ -1074,6 +1148,16 @@ sealed class BroadcastUiState {
*/ */
const val SPEAKING_TIMEOUT_MS: Long = 250L const val SPEAKING_TIMEOUT_MS: Long = 250L
/**
* Coalescing interval for [NestViewModel.audioLevels]. The decode loop
* pushes a fresh peak every ~20 ms (one per Opus frame); we publish to
* the StateFlow at this cadence instead so the UI ring animates ~10 Hz
* instead of recomposing every frame. 100 ms is fast enough that the
* eye still reads the throb as live, slow enough that the cost across
* a busy stage stays trivial.
*/
const val LEVEL_TICK_MS: Long = 100L
/** /**
* How long a kind-7 reaction stays in * How long a kind-7 reaction stays in
* [NestViewModel.recentReactions] before the eviction sweep * [NestViewModel.recentReactions] before the eviction sweep
@@ -56,10 +56,19 @@ class NestPlayer(
* *
* Decoder errors are reported via [onError] but do NOT stop the loop — * Decoder errors are reported via [onError] but do NOT stop the loop —
* one bad packet shouldn't tear down the room. Player errors are fatal. * one bad packet shouldn't tear down the room. Player errors are fatal.
*
* [onLevel] receives the peak amplitude of each successfully decoded
* frame, normalized to `[0, 1]`. Default no-op so callers that don't
* care about levels (tests, audio-only consumers) pay zero cost.
* Invoked on the same dispatcher as the decode loop — typically
* `viewModelScope`'s Main, so the consumer must keep its handler
* lightweight (e.g. a HashMap put followed by a coalesced StateFlow
* emission).
*/ */
fun play( fun play(
objects: Flow<MoqObject>, objects: Flow<MoqObject>,
onError: (AudioException) -> Unit = { /* swallow */ }, onError: (AudioException) -> Unit = { /* swallow */ },
onLevel: (Float) -> Unit = { /* no-op */ },
) { ) {
check(!stopped) { "NestPlayer already stopped" } check(!stopped) { "NestPlayer already stopped" }
check(job == null) { "NestPlayer.play already called" } check(job == null) { "NestPlayer.play already called" }
@@ -85,6 +94,7 @@ class NestPlayer(
return@collect return@collect
} }
if (pcm.isNotEmpty()) { if (pcm.isNotEmpty()) {
onLevel(peakAmplitude(pcm))
player.enqueue(pcm) player.enqueue(pcm)
} }
} }
@@ -102,6 +112,20 @@ class NestPlayer(
} }
} }
/**
* Peak amplitude of a 16-bit PCM frame, normalized to `[0, 1]`. Peak
* (vs RMS) is jittery on its own but responds instantly to onsets,
* which suits a visual ring that the UI smooths via animateDpAsState.
*/
private fun peakAmplitude(pcm: ShortArray): Float {
var maxAbs = 0
for (s in pcm) {
val abs = if (s.toInt() < 0) -s.toInt() else s.toInt()
if (abs > maxAbs) maxAbs = abs
}
return (maxAbs / 32768f).coerceIn(0f, 1f)
}
/** /**
* Stop playback, cancel the decode loop, release the decoder. Idempotent. * Stop playback, cancel the decode loop, release the decoder. Idempotent.
* *
@@ -150,6 +150,50 @@ class NestPlayerTest {
sut.stop() sut.stop()
} }
@Test
fun on_level_reports_normalized_peak_per_decoded_frame() =
runTest {
// Decode each input byte to a single 16-bit sample with a
// known peak so we can assert the level math directly.
// 0x01 -> 1/32768 (~0), 0x40 -> 0x4000 (0.5), 0x7F -> 0x7FFF (max).
val decoder = FakeOpusDecoder { bytes -> ShortArray(1) { (bytes[0].toInt() shl 8).toShort() } }
val player = FakeAudioPlayer()
val levels = mutableListOf<Float>()
val sut = NestPlayer(decoder, player, this)
sut.play(
objects =
flowOf(
moqObject(byteArrayOf(0x40)),
moqObject(byteArrayOf(0x7F)),
),
onLevel = { levels.add(it) },
)
testScheduler.advanceUntilIdle()
assertEquals(2, levels.size)
// 0x40 << 8 = 0x4000 = 16384 → 16384/32768 = 0.5
assertTrue(levels[0] in 0.49f..0.51f, "expected ~0.5, got ${levels[0]}")
// 0x7F << 8 = 0x7F00 → 0x7F00/32768 ≈ 0.992
assertTrue(levels[1] > 0.98f, "expected near-max, got ${levels[1]}")
sut.stop()
}
@Test
fun on_level_is_skipped_when_decoder_returns_empty_pcm() =
runTest {
val decoder = FakeOpusDecoder { ShortArray(0) }
val levels = mutableListOf<Float>()
val sut = NestPlayer(decoder, FakeAudioPlayer(), this)
sut.play(
objects = flowOf(moqObject(byteArrayOf(0x01))),
onLevel = { levels.add(it) },
)
testScheduler.advanceUntilIdle()
assertEquals(0, levels.size)
sut.stop()
}
@Test @Test
fun objects_arriving_after_play_are_streamed_through_the_pipeline() = fun objects_arriving_after_play_are_streamed_through_the_pipeline() =
runTest { runTest {