feat(media): VLCJ video playback with DirectRendering and player pool

Phase 4: VLCJ player pool (max 3 instances), DirectRendering via
CallbackVideoSurface → Skia Bitmap → Compose Image. Video controls
with auto-hide, seek, play/pause. Graceful fallback when VLC missing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
nrobi144
2026-03-16 10:53:04 +02:00
parent 2181407647
commit 405601463e
5 changed files with 615 additions and 5 deletions
@@ -75,6 +75,7 @@ import com.vitorpamplona.amethyst.desktop.model.DesktopIAccount
import com.vitorpamplona.amethyst.desktop.network.DefaultRelays
import com.vitorpamplona.amethyst.desktop.network.DesktopRelayConnectionManager
import com.vitorpamplona.amethyst.desktop.service.images.DesktopImageLoaderSetup
import com.vitorpamplona.amethyst.desktop.service.media.VlcjPlayerPool
import com.vitorpamplona.amethyst.desktop.subscriptions.DesktopRelaySubscriptionsCoordinator
import com.vitorpamplona.amethyst.desktop.ui.ComposeNoteDialog
import com.vitorpamplona.amethyst.desktop.ui.ConnectingRelaysScreen
@@ -140,6 +141,7 @@ sealed class DesktopScreen {
fun main() {
DesktopImageLoaderSetup.setup()
Runtime.getRuntime().addShutdownHook(Thread { VlcjPlayerPool.shutdown() })
application {
val windowState =
rememberWindowState(
@@ -0,0 +1,142 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.desktop.service.media
import uk.co.caprica.vlcj.factory.MediaPlayerFactory
import uk.co.caprica.vlcj.player.base.MediaPlayerEventAdapter
import uk.co.caprica.vlcj.player.embedded.EmbeddedMediaPlayer
import uk.co.caprica.vlcj.player.embedded.videosurface.VideoSurface
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.BufferFormatCallback
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.RenderCallback
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicBoolean
/**
* Manages a pool of VLCJ media players to avoid costly create/destroy cycles.
* Keeps strong references to prevent GC crashes from native callbacks.
*
* IMPORTANT: Never let player instances be garbage collected while native
* callbacks are active — this causes JVM segfaults.
*/
object VlcjPlayerPool {
private val available = AtomicBoolean(false)
private var factory: MediaPlayerFactory? = null
// Strong references to ALL created players (prevents GC crash)
private val allPlayers = mutableListOf<EmbeddedMediaPlayer>()
private val idlePlayers = ConcurrentLinkedQueue<EmbeddedMediaPlayer>()
private const val MAX_POOL_SIZE = 3
/**
* Initialize the pool. Returns false if VLC is not installed.
*/
fun init(): Boolean {
if (available.get()) return true
return try {
val f = MediaPlayerFactory("--no-xlib")
factory = f
available.set(true)
true
} catch (_: Exception) {
available.set(false)
false
}
}
fun isAvailable(): Boolean = available.get()
/**
* Create a callback video surface using the factory's API.
*/
fun createVideoSurface(
bufferFormatCallback: BufferFormatCallback,
renderCallback: RenderCallback,
): VideoSurface? {
val f = factory ?: return null
return f.videoSurfaces().newVideoSurface(bufferFormatCallback, renderCallback, true)
}
/**
* Acquire a player from the pool or create a new one.
* Returns null if VLC is not available or pool is at capacity.
*/
fun acquire(): EmbeddedMediaPlayer? {
if (!available.get()) return null
val f = factory ?: return null
// Reuse idle player
idlePlayers.poll()?.let { return it }
// Create new if under limit
synchronized(allPlayers) {
if (allPlayers.size >= MAX_POOL_SIZE) return null
return try {
val player = f.mediaPlayers().newEmbeddedMediaPlayer()
allPlayers.add(player)
player
} catch (_: Exception) {
null
}
}
}
/**
* Return a player to the pool for reuse. Stops playback first.
*/
fun release(player: EmbeddedMediaPlayer) {
try {
player.controls().stop()
// Remove any event listeners to prevent stale callbacks
player.events().addMediaPlayerEventListener(
object : MediaPlayerEventAdapter() {},
)
idlePlayers.offer(player)
} catch (_: Exception) {
// Player may already be disposed
}
}
/**
* Shut down the entire pool. Call on app exit.
*/
fun shutdown() {
synchronized(allPlayers) {
idlePlayers.clear()
for (player in allPlayers) {
try {
player.controls().stop()
player.release()
} catch (_: Exception) {
// Ignore
}
}
allPlayers.clear()
}
try {
factory?.release()
} catch (_: Exception) {
// Ignore
}
factory = null
available.set(false)
}
}
@@ -0,0 +1,268 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.desktop.ui.media
import androidx.compose.foundation.Image
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.aspectRatio
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.mutableLongStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.toComposeImageBitmap
import androidx.compose.ui.unit.dp
import com.vitorpamplona.amethyst.desktop.service.media.VlcjPlayerPool
import kotlinx.coroutines.delay
import org.jetbrains.skia.Bitmap
import org.jetbrains.skia.ColorAlphaType
import org.jetbrains.skia.ImageInfo
import uk.co.caprica.vlcj.player.base.MediaPlayer
import uk.co.caprica.vlcj.player.base.MediaPlayerEventAdapter
import uk.co.caprica.vlcj.player.embedded.EmbeddedMediaPlayer
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.BufferFormat
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.BufferFormatCallback
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.RenderCallback
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.format.RV32BufferFormat
import java.nio.ByteBuffer
import org.jetbrains.skia.Image as SkiaImage
data class VideoPlayerState(
val isPlaying: Boolean = false,
val position: Float = 0f,
val duration: Long = 0L,
val currentTime: Long = 0L,
)
@Composable
fun DesktopVideoPlayer(
url: String,
modifier: Modifier = Modifier,
autoPlay: Boolean = false,
) {
var frame by remember { mutableStateOf<ImageBitmap?>(null) }
var isPlaying by remember { mutableStateOf(false) }
var position by remember { mutableFloatStateOf(0f) }
var duration by remember { mutableLongStateOf(0L) }
var currentTime by remember { mutableLongStateOf(0L) }
var aspectRatio by remember { mutableFloatStateOf(16f / 9f) }
var vlcAvailable by remember { mutableStateOf(true) }
var player by remember { mutableStateOf<EmbeddedMediaPlayer?>(null) }
// Acquire player
DisposableEffect(url) {
if (!VlcjPlayerPool.init()) {
vlcAvailable = false
return@DisposableEffect onDispose {}
}
val acquired = VlcjPlayerPool.acquire()
if (acquired == null) {
vlcAvailable = false
return@DisposableEffect onDispose {}
}
// Skia bitmap for DirectRendering
var skBitmap: Bitmap? = null
var videoWidth = 0
var videoHeight = 0
val bufferFormatCallback =
object : BufferFormatCallback {
override fun getBufferFormat(
sourceWidth: Int,
sourceHeight: Int,
): BufferFormat {
videoWidth = sourceWidth
videoHeight = sourceHeight
if (sourceHeight > 0) {
aspectRatio = sourceWidth.toFloat() / sourceHeight.toFloat()
}
// Allocate Skia bitmap
val bmp = Bitmap()
bmp.allocPixels(
ImageInfo.makeN32(sourceWidth, sourceHeight, ColorAlphaType.PREMUL),
)
skBitmap = bmp
return RV32BufferFormat(sourceWidth, sourceHeight)
}
override fun allocatedBuffers(buffers: Array<out ByteBuffer>) {
// No-op: we copy from the buffer in render callback
}
}
val renderCallback =
RenderCallback { _, nativeBuffers, _ ->
val bmp = skBitmap ?: return@RenderCallback
val buffer = nativeBuffers[0]
buffer.rewind()
val bytes = ByteArray(buffer.remaining())
buffer.get(bytes)
bmp.installPixels(bytes)
frame = SkiaImage.makeFromBitmap(bmp).toComposeImageBitmap()
}
val surface =
VlcjPlayerPool.createVideoSurface(
bufferFormatCallback,
renderCallback,
)
acquired.videoSurface().set(surface)
acquired.events().addMediaPlayerEventListener(
object : MediaPlayerEventAdapter() {
override fun playing(mediaPlayer: MediaPlayer) {
isPlaying = true
duration = mediaPlayer.status().length()
}
override fun paused(mediaPlayer: MediaPlayer) {
isPlaying = false
}
override fun stopped(mediaPlayer: MediaPlayer) {
isPlaying = false
}
override fun positionChanged(
mediaPlayer: MediaPlayer,
newPosition: Float,
) {
position = newPosition
currentTime = (newPosition * duration).toLong()
}
override fun finished(mediaPlayer: MediaPlayer) {
isPlaying = false
position = 0f
currentTime = 0L
}
},
)
player = acquired
if (autoPlay) {
acquired.media().play(url)
} else {
acquired.media().prepare(url)
}
onDispose {
player = null
VlcjPlayerPool.release(acquired)
}
}
// Position polling (VLCJ events sometimes miss updates)
LaunchedEffect(isPlaying) {
while (isPlaying) {
delay(500)
player?.let {
position = it.status().position()
currentTime = it.status().time()
}
}
}
if (!vlcAvailable) {
VlcNotAvailableMessage(url, modifier)
return
}
Box(
modifier =
modifier
.fillMaxWidth()
.aspectRatio(aspectRatio)
.clip(RoundedCornerShape(8.dp))
.background(MaterialTheme.colorScheme.surfaceContainerHigh),
contentAlignment = Alignment.Center,
) {
frame?.let { bitmap ->
Image(
bitmap = bitmap,
contentDescription = "Video",
modifier = Modifier.fillMaxWidth(),
)
}
VideoControls(
isPlaying = isPlaying,
position = position,
duration = duration,
currentTime = currentTime,
onPlayPause = {
player?.let { p ->
if (isPlaying) {
p.controls().pause()
} else {
if (position <= 0f && !p.status().isPlaying) {
p.media().play(url)
} else {
p.controls().play()
}
}
}
},
onSeek = { pos ->
player?.controls()?.setPosition(pos)
},
)
}
}
@Composable
private fun VlcNotAvailableMessage(
url: String,
modifier: Modifier = Modifier,
) {
Box(
modifier =
modifier
.fillMaxWidth()
.clip(RoundedCornerShape(8.dp))
.background(MaterialTheme.colorScheme.surfaceContainerHigh),
contentAlignment = Alignment.Center,
) {
Text(
text = "Video: $url\nInstall VLC to play videos: https://www.videolan.org/vlc/",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
modifier = Modifier.fillMaxWidth(),
)
}
}
@@ -0,0 +1,177 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.desktop.ui.media
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.foundation.background
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Pause
import androidx.compose.material.icons.filled.PlayArrow
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Slider
import androidx.compose.material3.SliderDefaults
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.input.pointer.PointerEventType
import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.unit.dp
import kotlinx.coroutines.delay
@Composable
fun VideoControls(
isPlaying: Boolean,
position: Float,
duration: Long,
currentTime: Long,
onPlayPause: () -> Unit,
onSeek: (Float) -> Unit,
modifier: Modifier = Modifier,
) {
var showControls by remember { mutableStateOf(true) }
var hovering by remember { mutableStateOf(false) }
// Auto-hide controls after 2s when playing
LaunchedEffect(isPlaying, hovering) {
if (isPlaying && !hovering) {
delay(2000)
showControls = false
} else {
showControls = true
}
}
Box(
modifier =
modifier
.fillMaxSize()
.clickable { onPlayPause() }
.pointerInput(Unit) {
awaitPointerEventScope {
while (true) {
val event = awaitPointerEvent()
when (event.type) {
PointerEventType.Enter -> {
hovering = true
showControls = true
}
PointerEventType.Exit -> {
hovering = false
}
}
}
}
},
) {
// Center play button when paused
if (!isPlaying) {
IconButton(
onClick = onPlayPause,
modifier = Modifier.align(Alignment.Center).size(64.dp),
) {
Icon(
Icons.Default.PlayArrow,
contentDescription = "Play",
tint = Color.White,
modifier = Modifier.size(48.dp),
)
}
}
// Bottom controls bar
AnimatedVisibility(
visible = showControls,
enter = fadeIn(),
exit = fadeOut(),
modifier = Modifier.align(Alignment.BottomCenter),
) {
Row(
modifier =
Modifier
.fillMaxWidth()
.background(Color.Black.copy(alpha = 0.5f))
.padding(horizontal = 8.dp, vertical = 4.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
IconButton(onClick = onPlayPause, modifier = Modifier.size(32.dp)) {
Icon(
if (isPlaying) Icons.Default.Pause else Icons.Default.PlayArrow,
contentDescription = if (isPlaying) "Pause" else "Play",
tint = Color.White,
modifier = Modifier.size(20.dp),
)
}
Text(
text = formatTime(currentTime),
style = MaterialTheme.typography.labelSmall,
color = Color.White,
)
Slider(
value = position,
onValueChange = onSeek,
modifier = Modifier.weight(1f),
colors =
SliderDefaults.colors(
thumbColor = Color.White,
activeTrackColor = MaterialTheme.colorScheme.primary,
inactiveTrackColor = Color.White.copy(alpha = 0.3f),
),
)
Text(
text = formatTime(duration),
style = MaterialTheme.typography.labelSmall,
color = Color.White,
)
}
}
}
}
private fun formatTime(millis: Long): String {
val totalSeconds = millis / 1000
val minutes = totalSeconds / 60
val seconds = totalSeconds % 60
return "%d:%02d".format(minutes, seconds)
}
@@ -54,6 +54,7 @@ import com.vitorpamplona.amethyst.commons.richtext.UrlParser
import com.vitorpamplona.amethyst.commons.richtext.Urls
import com.vitorpamplona.amethyst.commons.ui.components.UserAvatar
import com.vitorpamplona.amethyst.commons.util.toTimeAgo
import com.vitorpamplona.amethyst.desktop.ui.media.DesktopVideoPlayer
/**
* Data class for displaying a note card.
@@ -83,19 +84,23 @@ fun NoteCard(
remember(urls) {
urls.withScheme.filter { RichTextParser.isImageUrl(it) }
}
val videoUrls =
remember(urls) {
urls.withScheme.filter { RichTextParser.isVideoUrl(it) }
}
val mediaUrls = remember(imageUrls, videoUrls) { (imageUrls + videoUrls).toSet() }
val strippedContent =
remember(note.content, imageUrls) {
remember(note.content, mediaUrls) {
var text = note.content
for (url in imageUrls) {
for (url in mediaUrls) {
text = text.replace(url, "").trim()
}
text
}
val strippedUrls =
remember(urls, imageUrls) {
val imageSet = imageUrls.toSet()
remember(urls, mediaUrls) {
Urls(
withScheme = urls.withScheme - imageSet,
withScheme = urls.withScheme - mediaUrls,
withoutScheme = urls.withoutScheme,
emails = urls.emails,
bech32s = urls.bech32s,
@@ -185,6 +190,22 @@ fun NoteCard(
}
}
// Inline videos
if (videoUrls.isNotEmpty()) {
if (strippedContent.isNotBlank() || imageUrls.isNotEmpty()) {
Spacer(Modifier.height(8.dp))
}
for (url in videoUrls) {
DesktopVideoPlayer(
url = url,
modifier = Modifier.fillMaxWidth().heightIn(max = 400.dp),
)
if (url != videoUrls.last()) {
Spacer(Modifier.height(4.dp))
}
}
}
Spacer(Modifier.height(8.dp))
HorizontalDivider(color = MaterialTheme.colorScheme.outline.copy(alpha = 0.3f))