From 8de148e56248b96b80fa9c86ce8aacfa6f1df1ce Mon Sep 17 00:00:00 2001 From: davotoula Date: Mon, 12 Jan 2026 11:59:53 +0100 Subject: [PATCH] add VoiceAnonymizer for audio pitch/formant shifting --- .../ui/actions/uploads/VoiceAnonymizer.kt | 435 ++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/actions/uploads/VoiceAnonymizer.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/actions/uploads/VoiceAnonymizer.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/actions/uploads/VoiceAnonymizer.kt new file mode 100644 index 000000000..08d15c859 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/actions/uploads/VoiceAnonymizer.kt @@ -0,0 +1,435 @@ +/** + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.actions.uploads + +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaMuxer +import android.util.Log +import be.tarsos.dsp.AudioDispatcher +import be.tarsos.dsp.AudioEvent +import be.tarsos.dsp.AudioProcessor +import be.tarsos.dsp.WaveformSimilarityBasedOverlapAdd +import be.tarsos.dsp.io.TarsosDSPAudioFloatConverter +import be.tarsos.dsp.io.TarsosDSPAudioFormat +import be.tarsos.dsp.resample.RateTransposer +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.isActive +import kotlinx.coroutines.withContext +import java.io.File +import java.nio.ByteOrder +import kotlin.coroutines.coroutineContext +import kotlin.math.abs + +data class AnonymizedResult( + val file: File, + val waveform: List, + val duration: Int, +) + +class VoiceAnonymizer { + companion object { + private const val TAG = "VoiceAnonymizer" + private const val SAMPLE_RATE = 44100 + private const val CHANNELS = 1 + private const val BIT_RATE = 128000 + } + + suspend fun anonymize( + inputFile: File, + preset: VoicePreset, + onProgress: (Float) -> Unit = {}, + ): Result = + withContext(Dispatchers.IO) { + if (preset == VoicePreset.NONE) { + return@withContext Result.failure( + IllegalArgumentException("Cannot anonymize with NONE preset"), + ) + } + + try { + val outputFile = createOutputFile(inputFile, preset) + val (pcmData, sampleRate, duration) = + decodeAudioToPcm(inputFile) { progress -> + onProgress(progress * 0.3f) + } + + val processedPcm = + processPcmWithTarsos(pcmData, preset, sampleRate) { progress -> + onProgress(0.3f + progress * 0.4f) + } + + val waveform = extractWaveform(processedPcm, sampleRate) + + encodePcmToAac(processedPcm, sampleRate, outputFile) { progress -> + onProgress(0.7f + progress * 0.3f) + } + + onProgress(1f) + Result.success(AnonymizedResult(outputFile, waveform, duration)) + } catch (e: Exception) { + Log.e(TAG, "Failed to anonymize audio", e) + Result.failure(e) + } + } + + private fun createOutputFile( + inputFile: File, + preset: VoicePreset, + ): File { + val baseName = inputFile.nameWithoutExtension + val presetSuffix = preset.name.lowercase() + return File(inputFile.parentFile, "${baseName}_$presetSuffix.mp4") + } + + private data class DecodedAudio( + val pcmData: FloatArray, + val sampleRate: Int, + val duration: Int, + ) + + private suspend fun decodeAudioToPcm( + inputFile: File, + onProgress: (Float) -> Unit, + ): DecodedAudio { + val extractor = MediaExtractor() + extractor.setDataSource(inputFile.absolutePath) + + var audioTrackIndex = -1 + var format: MediaFormat? = null + for (i in 0 until extractor.trackCount) { + val trackFormat = extractor.getTrackFormat(i) + val mime = trackFormat.getString(MediaFormat.KEY_MIME) + if (mime?.startsWith("audio/") == true) { + audioTrackIndex = i + format = trackFormat + break + } + } + + if (audioTrackIndex == -1 || format == null) { + extractor.release() + throw IllegalStateException("No audio track found in file") + } + + extractor.selectTrack(audioTrackIndex) + val mime = format.getString(MediaFormat.KEY_MIME) ?: "audio/mp4a-latm" + val sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE) + val durationUs = format.getLong(MediaFormat.KEY_DURATION) + val duration = (durationUs / 1_000_000).toInt() + + val decoder = MediaCodec.createDecoderByType(mime) + decoder.configure(format, null, null, 0) + decoder.start() + + val pcmSamples = mutableListOf() + val bufferInfo = MediaCodec.BufferInfo() + var inputDone = false + var outputDone = false + + while (!outputDone && coroutineContext.isActive) { + if (!inputDone) { + val inputBufferIndex = decoder.dequeueInputBuffer(10000) + if (inputBufferIndex >= 0) { + val inputBuffer = decoder.getInputBuffer(inputBufferIndex)!! + val sampleSize = extractor.readSampleData(inputBuffer, 0) + if (sampleSize < 0) { + decoder.queueInputBuffer( + inputBufferIndex, + 0, + 0, + 0, + MediaCodec.BUFFER_FLAG_END_OF_STREAM, + ) + inputDone = true + } else { + val presentationTimeUs = extractor.sampleTime + decoder.queueInputBuffer( + inputBufferIndex, + 0, + sampleSize, + presentationTimeUs, + 0, + ) + extractor.advance() + if (durationUs > 0) { + onProgress((presentationTimeUs.toFloat() / durationUs).coerceIn(0f, 1f)) + } + } + } + } + + val outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000) + if (outputBufferIndex >= 0) { + val outputBuffer = decoder.getOutputBuffer(outputBufferIndex)!! + val shortBuffer = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer() + while (shortBuffer.hasRemaining()) { + pcmSamples.add(shortBuffer.get() / 32768f) + } + decoder.releaseOutputBuffer(outputBufferIndex, false) + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { + outputDone = true + } + } + } + + decoder.stop() + decoder.release() + extractor.release() + + return DecodedAudio(pcmSamples.toFloatArray(), sampleRate, duration) + } + + private fun processPcmWithTarsos( + pcmData: FloatArray, + preset: VoicePreset, + sampleRate: Int, + onProgress: (Float) -> Unit, + ): FloatArray { + val factor = preset.pitchFactor + val processedSamples = mutableListOf() + val totalSamples = pcmData.size + + val wsola = + WaveformSimilarityBasedOverlapAdd( + WaveformSimilarityBasedOverlapAdd.Parameters.musicDefaults( + factor, + sampleRate.toDouble(), + ), + ) + val rateTransposer = RateTransposer(factor) + + val bufferSize = wsola.inputBufferSize + val overlap = wsola.overlap + + val tarsosDspFormat = + TarsosDSPAudioFormat( + sampleRate.toFloat(), + 16, + 1, + true, + false, + ) + + val collector = + object : AudioProcessor { + override fun process(audioEvent: AudioEvent): Boolean { + val buffer = audioEvent.floatBuffer + for (i in 0 until audioEvent.bufferSize) { + processedSamples.add(buffer[i]) + } + return true + } + + override fun processingFinished() {} + } + + val dispatcher = + AudioDispatcher( + FloatArrayAudioInputStream(pcmData, tarsosDspFormat, pcmData.size.toLong()), + bufferSize, + overlap, + ) + + wsola.setDispatcher(dispatcher) + dispatcher.addAudioProcessor(wsola) + dispatcher.addAudioProcessor(rateTransposer) + dispatcher.addAudioProcessor(collector) + + var samplesProcessed = 0 + val progressProcessor = + object : AudioProcessor { + override fun process(audioEvent: AudioEvent): Boolean { + samplesProcessed += audioEvent.bufferSize + onProgress((samplesProcessed.toFloat() / totalSamples).coerceIn(0f, 1f)) + return true + } + + override fun processingFinished() {} + } + dispatcher.addAudioProcessor(progressProcessor) + + dispatcher.run() + + return processedSamples.toFloatArray() + } + + private fun extractWaveform( + pcmData: FloatArray, + sampleRate: Int, + ): List { + val waveform = mutableListOf() + var offset = 0 + + while (offset < pcmData.size) { + val end = minOf(offset + sampleRate, pcmData.size) + var maxAmplitude = 0f + for (i in offset until end) { + val amplitude = abs(pcmData[i]) + if (amplitude > maxAmplitude) { + maxAmplitude = amplitude + } + } + waveform.add(maxAmplitude * 32768f) + offset += sampleRate + } + + return waveform + } + + private fun encodePcmToAac( + pcmData: FloatArray, + sampleRate: Int, + outputFile: File, + onProgress: (Float) -> Unit, + ) { + val format = + MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, CHANNELS) + format.setInteger( + MediaFormat.KEY_AAC_PROFILE, + MediaCodecInfo.CodecProfileLevel.AACObjectLC, + ) + format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE) + + val encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) + encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + encoder.start() + + val muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + var audioTrackIndex = -1 + var muxerStarted = false + + val bufferInfo = MediaCodec.BufferInfo() + var inputOffset = 0 + var inputDone = false + var outputDone = false + val totalSamples = pcmData.size + + while (!outputDone) { + if (!inputDone) { + val inputBufferIndex = encoder.dequeueInputBuffer(10000) + if (inputBufferIndex >= 0) { + val inputBuffer = encoder.getInputBuffer(inputBufferIndex)!! + inputBuffer.clear() + + val samplesToWrite = minOf((inputBuffer.capacity() / 2), pcmData.size - inputOffset) + if (samplesToWrite <= 0) { + encoder.queueInputBuffer( + inputBufferIndex, + 0, + 0, + 0, + MediaCodec.BUFFER_FLAG_END_OF_STREAM, + ) + inputDone = true + } else { + for (i in 0 until samplesToWrite) { + val sample = + (pcmData[inputOffset + i] * 32767) + .toInt() + .coerceIn(-32768, 32767) + .toShort() + inputBuffer.putShort(sample) + } + val presentationTimeUs = (inputOffset * 1_000_000L) / sampleRate + encoder.queueInputBuffer( + inputBufferIndex, + 0, + inputBuffer.position(), + presentationTimeUs, + 0, + ) + inputOffset += samplesToWrite + onProgress(inputOffset.toFloat() / totalSamples) + } + } + } + + val outputBufferIndex = encoder.dequeueOutputBuffer(bufferInfo, 10000) + when { + outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + audioTrackIndex = muxer.addTrack(encoder.outputFormat) + muxer.start() + muxerStarted = true + } + + outputBufferIndex >= 0 -> { + val outputBuffer = encoder.getOutputBuffer(outputBufferIndex)!! + if (muxerStarted && bufferInfo.size > 0) { + outputBuffer.position(bufferInfo.offset) + outputBuffer.limit(bufferInfo.offset + bufferInfo.size) + muxer.writeSampleData(audioTrackIndex, outputBuffer, bufferInfo) + } + encoder.releaseOutputBuffer(outputBufferIndex, false) + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { + outputDone = true + } + } + } + } + + encoder.stop() + encoder.release() + muxer.stop() + muxer.release() + } +} + +private class FloatArrayAudioInputStream( + private val floatArray: FloatArray, + private val format: TarsosDSPAudioFormat, + private val frameLength: Long, +) : be.tarsos.dsp.io.TarsosDSPAudioInputStream { + private var position = 0 + + override fun getFormat(): TarsosDSPAudioFormat = format + + override fun getFrameLength(): Long = frameLength + + override fun read( + buffer: ByteArray, + offset: Int, + length: Int, + ): Int { + val converter = TarsosDSPAudioFloatConverter.getConverter(format) + val floatBuffer = FloatArray(length / 2) + val samplesToRead = minOf(floatBuffer.size, floatArray.size - position) + + if (samplesToRead <= 0) return -1 + + System.arraycopy(floatArray, position, floatBuffer, 0, samplesToRead) + position += samplesToRead + + converter.toByteArray(floatBuffer, samplesToRead, buffer, offset) + return samplesToRead * 2 + } + + override fun skip(bytesToSkip: Long): Long { + val samplesToSkip = (bytesToSkip / 2).toInt() + val actualSkip = minOf(samplesToSkip, floatArray.size - position) + position += actualSkip + return actualSkip.toLong() * 2 + } + + override fun close() {} +}