add VoiceAnonymizer for audio pitch/formant shifting
This commit is contained in:
davotoula
+435
@@ -0,0 +1,435 @@
|
|||||||
|
/**
|
||||||
|
* Copyright (c) 2025 Vitor Pamplona
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
* this software and associated documentation files (the "Software"), to deal in
|
||||||
|
* the Software without restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||||
|
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
* subject to the following conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be included in all
|
||||||
|
* copies or substantial portions of the Software.
|
||||||
|
*
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||||
|
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
|
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
package com.vitorpamplona.amethyst.ui.actions.uploads
|
||||||
|
|
||||||
|
import android.media.MediaCodec
|
||||||
|
import android.media.MediaCodecInfo
|
||||||
|
import android.media.MediaExtractor
|
||||||
|
import android.media.MediaFormat
|
||||||
|
import android.media.MediaMuxer
|
||||||
|
import android.util.Log
|
||||||
|
import be.tarsos.dsp.AudioDispatcher
|
||||||
|
import be.tarsos.dsp.AudioEvent
|
||||||
|
import be.tarsos.dsp.AudioProcessor
|
||||||
|
import be.tarsos.dsp.WaveformSimilarityBasedOverlapAdd
|
||||||
|
import be.tarsos.dsp.io.TarsosDSPAudioFloatConverter
|
||||||
|
import be.tarsos.dsp.io.TarsosDSPAudioFormat
|
||||||
|
import be.tarsos.dsp.resample.RateTransposer
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.isActive
|
||||||
|
import kotlinx.coroutines.withContext
|
||||||
|
import java.io.File
|
||||||
|
import java.nio.ByteOrder
|
||||||
|
import kotlin.coroutines.coroutineContext
|
||||||
|
import kotlin.math.abs
|
||||||
|
|
||||||
|
data class AnonymizedResult(
|
||||||
|
val file: File,
|
||||||
|
val waveform: List<Float>,
|
||||||
|
val duration: Int,
|
||||||
|
)
|
||||||
|
|
||||||
|
class VoiceAnonymizer {
|
||||||
|
companion object {
|
||||||
|
private const val TAG = "VoiceAnonymizer"
|
||||||
|
private const val SAMPLE_RATE = 44100
|
||||||
|
private const val CHANNELS = 1
|
||||||
|
private const val BIT_RATE = 128000
|
||||||
|
}
|
||||||
|
|
||||||
|
suspend fun anonymize(
|
||||||
|
inputFile: File,
|
||||||
|
preset: VoicePreset,
|
||||||
|
onProgress: (Float) -> Unit = {},
|
||||||
|
): Result<AnonymizedResult> =
|
||||||
|
withContext(Dispatchers.IO) {
|
||||||
|
if (preset == VoicePreset.NONE) {
|
||||||
|
return@withContext Result.failure(
|
||||||
|
IllegalArgumentException("Cannot anonymize with NONE preset"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
val outputFile = createOutputFile(inputFile, preset)
|
||||||
|
val (pcmData, sampleRate, duration) =
|
||||||
|
decodeAudioToPcm(inputFile) { progress ->
|
||||||
|
onProgress(progress * 0.3f)
|
||||||
|
}
|
||||||
|
|
||||||
|
val processedPcm =
|
||||||
|
processPcmWithTarsos(pcmData, preset, sampleRate) { progress ->
|
||||||
|
onProgress(0.3f + progress * 0.4f)
|
||||||
|
}
|
||||||
|
|
||||||
|
val waveform = extractWaveform(processedPcm, sampleRate)
|
||||||
|
|
||||||
|
encodePcmToAac(processedPcm, sampleRate, outputFile) { progress ->
|
||||||
|
onProgress(0.7f + progress * 0.3f)
|
||||||
|
}
|
||||||
|
|
||||||
|
onProgress(1f)
|
||||||
|
Result.success(AnonymizedResult(outputFile, waveform, duration))
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Log.e(TAG, "Failed to anonymize audio", e)
|
||||||
|
Result.failure(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun createOutputFile(
|
||||||
|
inputFile: File,
|
||||||
|
preset: VoicePreset,
|
||||||
|
): File {
|
||||||
|
val baseName = inputFile.nameWithoutExtension
|
||||||
|
val presetSuffix = preset.name.lowercase()
|
||||||
|
return File(inputFile.parentFile, "${baseName}_$presetSuffix.mp4")
|
||||||
|
}
|
||||||
|
|
||||||
|
private data class DecodedAudio(
|
||||||
|
val pcmData: FloatArray,
|
||||||
|
val sampleRate: Int,
|
||||||
|
val duration: Int,
|
||||||
|
)
|
||||||
|
|
||||||
|
private suspend fun decodeAudioToPcm(
|
||||||
|
inputFile: File,
|
||||||
|
onProgress: (Float) -> Unit,
|
||||||
|
): DecodedAudio {
|
||||||
|
val extractor = MediaExtractor()
|
||||||
|
extractor.setDataSource(inputFile.absolutePath)
|
||||||
|
|
||||||
|
var audioTrackIndex = -1
|
||||||
|
var format: MediaFormat? = null
|
||||||
|
for (i in 0 until extractor.trackCount) {
|
||||||
|
val trackFormat = extractor.getTrackFormat(i)
|
||||||
|
val mime = trackFormat.getString(MediaFormat.KEY_MIME)
|
||||||
|
if (mime?.startsWith("audio/") == true) {
|
||||||
|
audioTrackIndex = i
|
||||||
|
format = trackFormat
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (audioTrackIndex == -1 || format == null) {
|
||||||
|
extractor.release()
|
||||||
|
throw IllegalStateException("No audio track found in file")
|
||||||
|
}
|
||||||
|
|
||||||
|
extractor.selectTrack(audioTrackIndex)
|
||||||
|
val mime = format.getString(MediaFormat.KEY_MIME) ?: "audio/mp4a-latm"
|
||||||
|
val sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE)
|
||||||
|
val durationUs = format.getLong(MediaFormat.KEY_DURATION)
|
||||||
|
val duration = (durationUs / 1_000_000).toInt()
|
||||||
|
|
||||||
|
val decoder = MediaCodec.createDecoderByType(mime)
|
||||||
|
decoder.configure(format, null, null, 0)
|
||||||
|
decoder.start()
|
||||||
|
|
||||||
|
val pcmSamples = mutableListOf<Float>()
|
||||||
|
val bufferInfo = MediaCodec.BufferInfo()
|
||||||
|
var inputDone = false
|
||||||
|
var outputDone = false
|
||||||
|
|
||||||
|
while (!outputDone && coroutineContext.isActive) {
|
||||||
|
if (!inputDone) {
|
||||||
|
val inputBufferIndex = decoder.dequeueInputBuffer(10000)
|
||||||
|
if (inputBufferIndex >= 0) {
|
||||||
|
val inputBuffer = decoder.getInputBuffer(inputBufferIndex)!!
|
||||||
|
val sampleSize = extractor.readSampleData(inputBuffer, 0)
|
||||||
|
if (sampleSize < 0) {
|
||||||
|
decoder.queueInputBuffer(
|
||||||
|
inputBufferIndex,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
MediaCodec.BUFFER_FLAG_END_OF_STREAM,
|
||||||
|
)
|
||||||
|
inputDone = true
|
||||||
|
} else {
|
||||||
|
val presentationTimeUs = extractor.sampleTime
|
||||||
|
decoder.queueInputBuffer(
|
||||||
|
inputBufferIndex,
|
||||||
|
0,
|
||||||
|
sampleSize,
|
||||||
|
presentationTimeUs,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
extractor.advance()
|
||||||
|
if (durationUs > 0) {
|
||||||
|
onProgress((presentationTimeUs.toFloat() / durationUs).coerceIn(0f, 1f))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000)
|
||||||
|
if (outputBufferIndex >= 0) {
|
||||||
|
val outputBuffer = decoder.getOutputBuffer(outputBufferIndex)!!
|
||||||
|
val shortBuffer = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer()
|
||||||
|
while (shortBuffer.hasRemaining()) {
|
||||||
|
pcmSamples.add(shortBuffer.get() / 32768f)
|
||||||
|
}
|
||||||
|
decoder.releaseOutputBuffer(outputBufferIndex, false)
|
||||||
|
if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
|
||||||
|
outputDone = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
decoder.stop()
|
||||||
|
decoder.release()
|
||||||
|
extractor.release()
|
||||||
|
|
||||||
|
return DecodedAudio(pcmSamples.toFloatArray(), sampleRate, duration)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun processPcmWithTarsos(
|
||||||
|
pcmData: FloatArray,
|
||||||
|
preset: VoicePreset,
|
||||||
|
sampleRate: Int,
|
||||||
|
onProgress: (Float) -> Unit,
|
||||||
|
): FloatArray {
|
||||||
|
val factor = preset.pitchFactor
|
||||||
|
val processedSamples = mutableListOf<Float>()
|
||||||
|
val totalSamples = pcmData.size
|
||||||
|
|
||||||
|
val wsola =
|
||||||
|
WaveformSimilarityBasedOverlapAdd(
|
||||||
|
WaveformSimilarityBasedOverlapAdd.Parameters.musicDefaults(
|
||||||
|
factor,
|
||||||
|
sampleRate.toDouble(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
val rateTransposer = RateTransposer(factor)
|
||||||
|
|
||||||
|
val bufferSize = wsola.inputBufferSize
|
||||||
|
val overlap = wsola.overlap
|
||||||
|
|
||||||
|
val tarsosDspFormat =
|
||||||
|
TarsosDSPAudioFormat(
|
||||||
|
sampleRate.toFloat(),
|
||||||
|
16,
|
||||||
|
1,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
|
||||||
|
val collector =
|
||||||
|
object : AudioProcessor {
|
||||||
|
override fun process(audioEvent: AudioEvent): Boolean {
|
||||||
|
val buffer = audioEvent.floatBuffer
|
||||||
|
for (i in 0 until audioEvent.bufferSize) {
|
||||||
|
processedSamples.add(buffer[i])
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun processingFinished() {}
|
||||||
|
}
|
||||||
|
|
||||||
|
val dispatcher =
|
||||||
|
AudioDispatcher(
|
||||||
|
FloatArrayAudioInputStream(pcmData, tarsosDspFormat, pcmData.size.toLong()),
|
||||||
|
bufferSize,
|
||||||
|
overlap,
|
||||||
|
)
|
||||||
|
|
||||||
|
wsola.setDispatcher(dispatcher)
|
||||||
|
dispatcher.addAudioProcessor(wsola)
|
||||||
|
dispatcher.addAudioProcessor(rateTransposer)
|
||||||
|
dispatcher.addAudioProcessor(collector)
|
||||||
|
|
||||||
|
var samplesProcessed = 0
|
||||||
|
val progressProcessor =
|
||||||
|
object : AudioProcessor {
|
||||||
|
override fun process(audioEvent: AudioEvent): Boolean {
|
||||||
|
samplesProcessed += audioEvent.bufferSize
|
||||||
|
onProgress((samplesProcessed.toFloat() / totalSamples).coerceIn(0f, 1f))
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun processingFinished() {}
|
||||||
|
}
|
||||||
|
dispatcher.addAudioProcessor(progressProcessor)
|
||||||
|
|
||||||
|
dispatcher.run()
|
||||||
|
|
||||||
|
return processedSamples.toFloatArray()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun extractWaveform(
|
||||||
|
pcmData: FloatArray,
|
||||||
|
sampleRate: Int,
|
||||||
|
): List<Float> {
|
||||||
|
val waveform = mutableListOf<Float>()
|
||||||
|
var offset = 0
|
||||||
|
|
||||||
|
while (offset < pcmData.size) {
|
||||||
|
val end = minOf(offset + sampleRate, pcmData.size)
|
||||||
|
var maxAmplitude = 0f
|
||||||
|
for (i in offset until end) {
|
||||||
|
val amplitude = abs(pcmData[i])
|
||||||
|
if (amplitude > maxAmplitude) {
|
||||||
|
maxAmplitude = amplitude
|
||||||
|
}
|
||||||
|
}
|
||||||
|
waveform.add(maxAmplitude * 32768f)
|
||||||
|
offset += sampleRate
|
||||||
|
}
|
||||||
|
|
||||||
|
return waveform
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun encodePcmToAac(
|
||||||
|
pcmData: FloatArray,
|
||||||
|
sampleRate: Int,
|
||||||
|
outputFile: File,
|
||||||
|
onProgress: (Float) -> Unit,
|
||||||
|
) {
|
||||||
|
val format =
|
||||||
|
MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, CHANNELS)
|
||||||
|
format.setInteger(
|
||||||
|
MediaFormat.KEY_AAC_PROFILE,
|
||||||
|
MediaCodecInfo.CodecProfileLevel.AACObjectLC,
|
||||||
|
)
|
||||||
|
format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE)
|
||||||
|
|
||||||
|
val encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
|
||||||
|
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
|
||||||
|
encoder.start()
|
||||||
|
|
||||||
|
val muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
|
||||||
|
var audioTrackIndex = -1
|
||||||
|
var muxerStarted = false
|
||||||
|
|
||||||
|
val bufferInfo = MediaCodec.BufferInfo()
|
||||||
|
var inputOffset = 0
|
||||||
|
var inputDone = false
|
||||||
|
var outputDone = false
|
||||||
|
val totalSamples = pcmData.size
|
||||||
|
|
||||||
|
while (!outputDone) {
|
||||||
|
if (!inputDone) {
|
||||||
|
val inputBufferIndex = encoder.dequeueInputBuffer(10000)
|
||||||
|
if (inputBufferIndex >= 0) {
|
||||||
|
val inputBuffer = encoder.getInputBuffer(inputBufferIndex)!!
|
||||||
|
inputBuffer.clear()
|
||||||
|
|
||||||
|
val samplesToWrite = minOf((inputBuffer.capacity() / 2), pcmData.size - inputOffset)
|
||||||
|
if (samplesToWrite <= 0) {
|
||||||
|
encoder.queueInputBuffer(
|
||||||
|
inputBufferIndex,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
MediaCodec.BUFFER_FLAG_END_OF_STREAM,
|
||||||
|
)
|
||||||
|
inputDone = true
|
||||||
|
} else {
|
||||||
|
for (i in 0 until samplesToWrite) {
|
||||||
|
val sample =
|
||||||
|
(pcmData[inputOffset + i] * 32767)
|
||||||
|
.toInt()
|
||||||
|
.coerceIn(-32768, 32767)
|
||||||
|
.toShort()
|
||||||
|
inputBuffer.putShort(sample)
|
||||||
|
}
|
||||||
|
val presentationTimeUs = (inputOffset * 1_000_000L) / sampleRate
|
||||||
|
encoder.queueInputBuffer(
|
||||||
|
inputBufferIndex,
|
||||||
|
0,
|
||||||
|
inputBuffer.position(),
|
||||||
|
presentationTimeUs,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
inputOffset += samplesToWrite
|
||||||
|
onProgress(inputOffset.toFloat() / totalSamples)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val outputBufferIndex = encoder.dequeueOutputBuffer(bufferInfo, 10000)
|
||||||
|
when {
|
||||||
|
outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
|
||||||
|
audioTrackIndex = muxer.addTrack(encoder.outputFormat)
|
||||||
|
muxer.start()
|
||||||
|
muxerStarted = true
|
||||||
|
}
|
||||||
|
|
||||||
|
outputBufferIndex >= 0 -> {
|
||||||
|
val outputBuffer = encoder.getOutputBuffer(outputBufferIndex)!!
|
||||||
|
if (muxerStarted && bufferInfo.size > 0) {
|
||||||
|
outputBuffer.position(bufferInfo.offset)
|
||||||
|
outputBuffer.limit(bufferInfo.offset + bufferInfo.size)
|
||||||
|
muxer.writeSampleData(audioTrackIndex, outputBuffer, bufferInfo)
|
||||||
|
}
|
||||||
|
encoder.releaseOutputBuffer(outputBufferIndex, false)
|
||||||
|
if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
|
||||||
|
outputDone = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
encoder.stop()
|
||||||
|
encoder.release()
|
||||||
|
muxer.stop()
|
||||||
|
muxer.release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class FloatArrayAudioInputStream(
|
||||||
|
private val floatArray: FloatArray,
|
||||||
|
private val format: TarsosDSPAudioFormat,
|
||||||
|
private val frameLength: Long,
|
||||||
|
) : be.tarsos.dsp.io.TarsosDSPAudioInputStream {
|
||||||
|
private var position = 0
|
||||||
|
|
||||||
|
override fun getFormat(): TarsosDSPAudioFormat = format
|
||||||
|
|
||||||
|
override fun getFrameLength(): Long = frameLength
|
||||||
|
|
||||||
|
override fun read(
|
||||||
|
buffer: ByteArray,
|
||||||
|
offset: Int,
|
||||||
|
length: Int,
|
||||||
|
): Int {
|
||||||
|
val converter = TarsosDSPAudioFloatConverter.getConverter(format)
|
||||||
|
val floatBuffer = FloatArray(length / 2)
|
||||||
|
val samplesToRead = minOf(floatBuffer.size, floatArray.size - position)
|
||||||
|
|
||||||
|
if (samplesToRead <= 0) return -1
|
||||||
|
|
||||||
|
System.arraycopy(floatArray, position, floatBuffer, 0, samplesToRead)
|
||||||
|
position += samplesToRead
|
||||||
|
|
||||||
|
converter.toByteArray(floatBuffer, samplesToRead, buffer, offset)
|
||||||
|
return samplesToRead * 2
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun skip(bytesToSkip: Long): Long {
|
||||||
|
val samplesToSkip = (bytesToSkip / 2).toInt()
|
||||||
|
val actualSkip = minOf(samplesToSkip, floatArray.size - position)
|
||||||
|
position += actualSkip
|
||||||
|
return actualSkip.toLong() * 2
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user