diff --git a/cli/hang-interop/hang-publish/src/main.rs b/cli/hang-interop/hang-publish/src/main.rs index 132d6a12b..fb4448437 100644 --- a/cli/hang-interop/hang-publish/src/main.rs +++ b/cli/hang-interop/hang-publish/src/main.rs @@ -44,18 +44,30 @@ struct Args { #[arg(long)] broadcast: String, - /// Sine-wave frequency in Hz (mono only). For stereo, see - /// `--freq-hz-right` once that lands. + /// Sine-wave frequency in Hz. Used as the default for every + /// channel; override per-channel via `--freq-hz-l` / `--freq-hz-r`. #[arg(long, default_value_t = 440)] freq_hz: u32, + /// Per-channel frequency override for the LEFT channel. Falls + /// back to `--freq-hz` when unset. + #[arg(long)] + freq_hz_l: Option, + + /// Per-channel frequency override for the RIGHT channel. + /// Ignored when `--channels 1`. Falls back to `--freq-hz` when + /// unset. + #[arg(long)] + freq_hz_r: Option, + /// Maximum runtime in seconds. #[arg(long, default_value_t = 5)] duration: u64, - /// Channel count: 1 (mono) or 2 (stereo). Stereo uses the same - /// frequency on both channels for now; per-channel frequency is - /// a Phase-2 follow-up. + /// Channel count: 1 (mono) or 2 (stereo). With `2` and + /// `--freq-hz-l` / `--freq-hz-r` set, the L/R channels carry + /// independent tones — useful for the I4 stereo cross-stack + /// scenario. #[arg(long, default_value_t = 1)] channels: u32, @@ -188,8 +200,19 @@ async fn publish(origin: &moq_lite::OriginProducer, args: &Args) -> anyhow::Resu .context("set opus bitrate")?; let total_frames = (args.duration * 1_000_000 / FRAME_DURATION_US) as usize; - let phase_step = - 2.0_f64 * std::f64::consts::PI * (args.freq_hz as f64) / (SAMPLE_RATE_HZ as f64); + // Per-channel phase step: each channel may have its own + // frequency (I4 stereo). Defaults to args.freq_hz on every + // channel. + let mut phase_steps: Vec = Vec::with_capacity(args.channels as usize); + for ch in 0..(args.channels as usize) { + let f = match (ch, args.freq_hz_l, args.freq_hz_r) { + (0, Some(l), _) => l, + (1, _, Some(r)) => r, + _ => args.freq_hz, + }; + phase_steps + .push(2.0_f64 * std::f64::consts::PI * (f as f64) / (SAMPLE_RATE_HZ as f64)); + } let mut sample_idx: u64 = 0; // Sized to libopus's worst-case output for one 20 ms frame. let mut opus_buf = vec![0u8; 4_000]; @@ -201,13 +224,14 @@ async fn publish(origin: &moq_lite::OriginProducer, args: &Args) -> anyhow::Resu let mut group: Option = None; for frame_no in 0..total_frames { - // Generate one PCM frame at the configured frequency. + // Generate one PCM frame, possibly with a different sine + // tone on each channel. let mut pcm = vec![0i16; FRAME_SIZE_SAMPLES * args.channels as usize]; for i in 0..FRAME_SIZE_SAMPLES { - let t = sample_idx + i as u64; - let v = ((t as f64) * phase_step).sin(); - let s = (v * 16_383.0) as i16; + let t = (sample_idx + i as u64) as f64; for ch in 0..(args.channels as usize) { + let v = (t * phase_steps[ch]).sin(); + let s = (v * 16_383.0) as i16; pcm[i * args.channels as usize + ch] = s; } } diff --git a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/JvmOpusDecoder.kt b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/JvmOpusDecoder.kt index 39aab8e45..c7a6b3339 100644 --- a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/JvmOpusDecoder.kt +++ b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/JvmOpusDecoder.kt @@ -32,7 +32,7 @@ import java.nio.ShortBuffer */ class JvmOpusDecoder( private val sampleRate: Int = AudioFormat.SAMPLE_RATE_HZ, - private val channelCount: Int = AudioFormat.CHANNELS, + private val channelCount: Int = AudioFormat.DEFAULT_CHANNELS, ) : OpusDecoder { private val handle: PointerByReference diff --git a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/JvmOpusEncoder.kt b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/JvmOpusEncoder.kt index 5f84969a9..2f6554b93 100644 --- a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/JvmOpusEncoder.kt +++ b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/JvmOpusEncoder.kt @@ -42,7 +42,7 @@ import java.nio.ShortBuffer */ class JvmOpusEncoder( private val sampleRate: Int = AudioFormat.SAMPLE_RATE_HZ, - private val channelCount: Int = AudioFormat.CHANNELS, + private val channelCount: Int = AudioFormat.DEFAULT_CHANNELS, targetBitrate: Int = DEFAULT_BITRATE_BPS, ) : OpusEncoder { private val handle: PointerByReference diff --git a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/PcmAssertions.kt b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/PcmAssertions.kt index 2c5cd9ca5..241d4f4d0 100644 --- a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/PcmAssertions.kt +++ b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/PcmAssertions.kt @@ -95,6 +95,41 @@ object PcmAssertions { } } + /** + * Stereo / multi-channel variant of [assertFftPeak]. [interleaved] + * is L/R/L/R/... (or N-channel interleaved); [expectedHzPerChannel] + * has one entry per channel and each per-channel slice is asserted + * independently. Used by the I4 stereo scenario where left = 440 + * and right = 660 — a regression that mixes channels (or sums + * them into mono) trips the per-channel FFT. + */ + fun assertFftPeakPerChannel( + interleaved: FloatArray, + expectedHzPerChannel: DoubleArray, + halfWindowHz: Double = 5.0, + sampleRate: Int = AudioFormat.SAMPLE_RATE_HZ, + ) { + val channels = expectedHzPerChannel.size + require(interleaved.size % channels == 0) { + "interleaved size ${interleaved.size} not divisible by channels=$channels" + } + val perChannelLen = interleaved.size / channels + for (ch in 0 until channels) { + val slice = FloatArray(perChannelLen) + for (i in 0 until perChannelLen) { + slice[i] = interleaved[i * channels + ch] + } + try { + assertFftPeak(slice, expectedHzPerChannel[ch], halfWindowHz, sampleRate) + } catch (t: Throwable) { + throw IllegalStateException( + "channel $ch (expected ${expectedHzPerChannel[ch]} Hz): ${t.message}", + t, + ) + } + } + } + /** * Zero crossings per second within ±[tolerance] (fractional) * of [expectedPerSecond]. Catches Opus predictor warble that diff --git a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/SineWaveAudioCapture.kt b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/SineWaveAudioCapture.kt index a7a99d45a..b524ee681 100644 --- a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/SineWaveAudioCapture.kt +++ b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/audio/SineWaveAudioCapture.kt @@ -37,13 +37,30 @@ import kotlin.math.sin * 50 million frames/sec instead of 50 frames/sec, fill the relay's * buffers, and surface as "no inboundSubs" frame drops. * - * Mono only (`channels = 1`) for Phase 1 — the I4 stereo scenario - * (Phase 2) extends this to a per-channel `freqHzL` / `freqHzR` pair. + * Defaults to mono (`channelCount = 1`). Stereo with per-channel + * frequencies — the I4 scenario uses 440 Hz left / 660 Hz right — + * is supported via [freqHzPerChannel]: pass an `IntArray` of size + * [channelCount] holding the desired per-channel frequency. If + * left null, every channel runs at [freqHz]. + * + * Output PCM is interleaved L/R/L/R/... for stereo — matches the + * format the Android `MediaCodecOpusEncoder` and our + * [JvmOpusEncoder] expect for stereo input. */ class SineWaveAudioCapture( private val freqHz: Int = 440, + private val channelCount: Int = 1, + private val freqHzPerChannel: IntArray? = null, private val amplitude: Short = 16_383, ) : AudioCapture { + init { + if (freqHzPerChannel != null) { + require(freqHzPerChannel.size == channelCount) { + "freqHzPerChannel.size (${freqHzPerChannel.size}) must equal channelCount ($channelCount)" + } + } + } + private var sampleIdx: Long = 0L /** Wallclock target for the next frame (`System.nanoTime` units). */ @@ -55,15 +72,21 @@ class SineWaveAudioCapture( override suspend fun readFrame(): ShortArray? { val samples = AudioFormat.FRAME_SIZE_SAMPLES - val out = ShortArray(samples) + val out = ShortArray(samples * channelCount) val baseIdx = sampleIdx - val angularStep = 2.0 * PI * freqHz / AudioFormat.SAMPLE_RATE_HZ + val twoPi = 2.0 * PI + val sampleRate = AudioFormat.SAMPLE_RATE_HZ.toDouble() for (i in 0 until samples) { - val v = (amplitude * sin(angularStep * (baseIdx + i))).toInt() - // Clamp defensively — amplitude is well below Short.MAX_VALUE - // by default, but a future bigger amplitude could otherwise - // wrap on the .toShort() truncation. - out[i] = v.coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort() + val t = (baseIdx + i).toDouble() + for (ch in 0 until channelCount) { + val freq = freqHzPerChannel?.get(ch) ?: freqHz + val v = (amplitude * sin(twoPi * freq * t / sampleRate)).toInt() + // Clamp defensively — amplitude is well below Short.MAX_VALUE + // by default, but a future bigger amplitude could otherwise + // wrap on the .toShort() truncation. + out[i * channelCount + ch] = + v.coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort() + } } sampleIdx = baseIdx + samples diff --git a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/interop/native/HangInteropTest.kt b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/interop/native/HangInteropTest.kt index 6b4c95052..03f337fe0 100644 --- a/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/interop/native/HangInteropTest.kt +++ b/nestsClient/src/jvmTest/kotlin/com/vitorpamplona/nestsclient/interop/native/HangInteropTest.kt @@ -20,13 +20,16 @@ */ package com.vitorpamplona.nestsclient.interop.native +import com.vitorpamplona.nestsclient.AudioBroadcastConfig import com.vitorpamplona.nestsclient.NestsClient import com.vitorpamplona.nestsclient.NestsRoomConfig import com.vitorpamplona.nestsclient.audio.AudioFormat +import com.vitorpamplona.nestsclient.audio.JvmOpusDecoder import com.vitorpamplona.nestsclient.audio.JvmOpusEncoder import com.vitorpamplona.nestsclient.audio.PcmAssertions import com.vitorpamplona.nestsclient.audio.SineWaveAudioCapture import com.vitorpamplona.nestsclient.buildRelayConnectTarget +import com.vitorpamplona.nestsclient.connectNestsListener import com.vitorpamplona.nestsclient.connectNestsSpeaker import com.vitorpamplona.nestsclient.connectReconnectingNestsSpeaker import com.vitorpamplona.nestsclient.moq.lite.MoqLiteSession @@ -473,6 +476,173 @@ class HangInteropTest { PcmAssertions.assertFftPeak(analysed, expectedHz = 440.0, halfWindowHz = 5.0) } + /** + * I4 forward — Amethyst Kotlin speaker broadcasts stereo Opus + * (L=440 Hz, R=660 Hz) to `hang-listen`. Asserts each channel's + * peak frequency independently, so a regression that downmixes + * to mono or swaps channels is caught. + * + * Validates the speaker-side stereo path end-to-end: + * - `AudioBroadcastConfig(channelCount = 2)` plumbs through + * `connectNestsSpeaker` → `MoqLiteNestsSpeaker` → catalog, + * - `MoqLiteHangCatalog.opus48k(name, 2)` emits + * `numberOfChannels: 2` in the published JSON, + * - `JvmOpusEncoder(channelCount = 2)` encodes interleaved + * L/R PCM into stereo Opus packets, + * - hang-listen's catalog reader picks up `channels = 2` and + * constructs an `opus::Decoder` for stereo, + * - the resulting Float32 PCM file is interleaved L/R/L/R/... + * with the per-channel tones intact. + */ + @Test + fun amethyst_speaker_to_hang_listener_stereo_440_660() = + runBlocking { + val out = + runSpeakerToHangListen( + speakerSeconds = 5, + captureFirstFrame = false, + channelCount = 2, + freqHzPerChannel = intArrayOf(440, 660), + ) + val pcm = readFloat32Pcm(out.pcmFile) + // Skip first 80 ms (40 ms × 2 channels = 80 ms of + // interleaved silence prefix). Opus look-ahead. + val warmup = AudioFormat.SAMPLE_RATE_HZ / 25 * 2 + val analysed = pcm.copyOfRange(warmup, pcm.size) + PcmAssertions.assertFftPeakPerChannel( + analysed, + expectedHzPerChannel = doubleArrayOf(440.0, 660.0), + halfWindowHz = 5.0, + ) + } + + /** + * I4 reverse — `hang-publish` broadcasts stereo Opus + * (L=440 Hz, R=660 Hz); the Amethyst Kotlin listener + * decodes via `JvmOpusDecoder(channelCount = 2)` and + * asserts the per-channel FFT peaks. Mirror of the I4 + * forward scenario for the listener-side stereo path. + * + * Validates: + * - hang-publish's `--freq-hz-l` / `--freq-hz-r` / + * `--channels 2` produce a real stereo Opus stream, + * - the Kotlin listener's `subscribeSpeaker` flow + * delivers stereo Opus packets (after timestamp + * strip), + * - `JvmOpusDecoder(channelCount = 2)` correctly + * interleaves L/R PCM, + * - per-channel FFT peaks are intact end-to-end. + */ + @Test + fun rust_hang_publish_stereo_to_kotlin_listener_440_660() = + runBlocking { + val harness = NativeMoqRelayHarness.shared() + val signer: NostrSigner = NostrSignerInternal(KeyPair()) + val pubkey = signer.pubKey + val room = + NestsRoomConfig( + authBaseUrl = "", + endpoint = harness.relayUrl, + hostPubkey = pubkey, + roomId = "rt-${UUID.randomUUID()}", + ) + val moqNamespace = room.moqNamespace() + + val pumpScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + val transport = + QuicWebTransportFactory( + parentScope = pumpScope, + certificateValidator = PermissiveCertificateValidator(), + ) + + // Spawn hang-publish under URL=, broadcast + // suffix=. The Kotlin listener subscribes to + // exactly that path via `subscribeSpeaker(pubkey)`. + val publishProc = + ProcessBuilder( + harness.hangPublishBin().toString(), + "--relay-url", + "${harness.relayUrl}/$moqNamespace", + "--broadcast", + pubkey, + "--track-name", + "audio/data", + "--channels", + "2", + "--freq-hz-l", + "440", + "--freq-hz-r", + "660", + "--duration", + "5", + ).redirectErrorStream(true) + .also { it.environment()["RUST_LOG"] = "info" } + .start() + + // Tiny breathing room so the publisher's ANNOUNCE + // Active is on the relay before we subscribe. + Thread.sleep(300) + + try { + val listener = + connectNestsListener( + httpClient = StaticTokenNestsClient, + transport = transport, + scope = pumpScope, + room = room, + signer = signer, + ) + val subscription = listener.subscribeSpeaker(pubkey) + val decoder = JvmOpusDecoder(channelCount = 2) + val pcm = mutableListOf() + try { + // Collect for ~4 s wallclock (publisher runs + // 5 s; allow tail buffer). The flow doesn't + // necessarily yield exactly N items — collect + // by time, not count. + withTimeoutOrNull(4_000L) { + subscription.objects.collect { obj -> + val samples = decoder.decode(obj.payload) + for (s in samples) pcm += s.toFloat() / Short.MAX_VALUE.toFloat() + } + } + } finally { + decoder.release() + listener.close() + } + + // Read publisher output BEFORE destroying so the + // stream is still open. Stops at EOF when the + // publisher exits naturally (5 s --duration), or + // returns whatever's been written so far if it's + // still running. + val published = + runCatching { + publishProc.inputStream.bufferedReader().readText() + }.getOrDefault("(stdout unavailable)") + publishProc.destroy() + assertTrue( + pcm.size >= AudioFormat.SAMPLE_RATE_HZ * 2, + "expected ≥ 1 s of stereo PCM (= 2× sample rate floats), " + + "got ${pcm.size} floats. hang-publish stderr:\n$published", + ) + + val pcmArr = pcm.toFloatArray() + // Skip first 80 ms (40 ms × 2 channels) — Opus look- + // ahead silence. + val warmup = AudioFormat.SAMPLE_RATE_HZ / 25 * 2 + val analysed = pcmArr.copyOfRange(warmup, pcmArr.size) + PcmAssertions.assertFftPeakPerChannel( + analysed, + expectedHzPerChannel = doubleArrayOf(440.0, 660.0), + halfWindowHz = 5.0, + ) + } finally { + pumpScope.coroutineContext[Job]?.cancel() + publishProc.destroy() + } + } + /** * Rust↔Rust round-trip: pure-Rust through our harness. * Validates the cargo workspace + relay config + `moq-lite-03` @@ -587,6 +757,12 @@ private suspend fun runSpeakerToHangListen( * the reconnect orchestrator. */ hotSwapAfterMs: Long? = null, + /** + * Per-channel sine-wave config for the speaker. Default mono + * 440 Hz; I4 stereo passes `(channels=2, freqHzPerChannel=[440, 660])`. + */ + channelCount: Int = 1, + freqHzPerChannel: IntArray? = null, ): HangListenOutput { val harness = NativeMoqRelayHarness.shared() @@ -656,6 +832,18 @@ private suspend fun runSpeakerToHangListen( certificateValidator = PermissiveCertificateValidator(), ) + val captureFactory: () -> SineWaveAudioCapture = { + SineWaveAudioCapture( + freqHz = 440, + channelCount = channelCount, + freqHzPerChannel = freqHzPerChannel, + ) + } + val encoderFactory: () -> JvmOpusEncoder = { + JvmOpusEncoder(channelCount = channelCount) + } + val broadcastConfig = AudioBroadcastConfig(channelCount = channelCount) + lateinit var listenProc: Process try { val speaker = @@ -667,8 +855,9 @@ private suspend fun runSpeakerToHangListen( room = room, signer = signer, speakerPubkeyHex = pubkey, - captureFactory = { SineWaveAudioCapture(freqHz = 440) }, - encoderFactory = { JvmOpusEncoder() }, + captureFactory = captureFactory, + encoderFactory = encoderFactory, + broadcastConfig = broadcastConfig, tokenRefreshAfterMs = hotSwapAfterMs, connector = { connectNestsSpeaker( @@ -678,8 +867,9 @@ private suspend fun runSpeakerToHangListen( room = room, signer = signer, speakerPubkeyHex = pubkey, - captureFactory = { SineWaveAudioCapture(freqHz = 440) }, - encoderFactory = { JvmOpusEncoder() }, + captureFactory = captureFactory, + encoderFactory = encoderFactory, + broadcastConfig = broadcastConfig, framesPerGroup = 5, ) }, @@ -692,8 +882,9 @@ private suspend fun runSpeakerToHangListen( room = room, signer = signer, speakerPubkeyHex = pubkey, - captureFactory = { SineWaveAudioCapture(freqHz = 440) }, - encoderFactory = { JvmOpusEncoder() }, + captureFactory = captureFactory, + encoderFactory = encoderFactory, + broadcastConfig = broadcastConfig, framesPerGroup = 5, ) }