fix(richtext): recognize HLS MIME types as video in createMediaContent + propagate imeta image field to MediaUrlVideo.artworkUri

Code review:
- simplify result types and shared helpers
- propagate CancellationException from poster path + cover sibling
This commit is contained in:
davotoula
2026-05-09 21:31:58 +02:00
parent 48d08f5895
commit 7a77e3f5ee
8 changed files with 215 additions and 54 deletions
@@ -63,8 +63,8 @@ object HlsKind1SiblingBuilder {
masterSha256: String?,
masterDimension: DimensionTag?,
posterUrl: String?,
blurhashValue: String?,
thumbhashValue: String?,
blurhash: String?,
thumbhash: String?,
createdAt: Long? = null,
): EventTemplate<TextNoteEvent> {
val content =
@@ -79,8 +79,8 @@ object HlsKind1SiblingBuilder {
masterSha256?.let { hash(it) }
masterDimension?.let { dims(it) }
posterUrl?.let { image(it) }
blurhashValue?.let { blurhash(it) }
thumbhashValue?.let { thumbhash(it) }
blurhash?.let { this.blurhash(it) }
thumbhash?.let { this.thumbhash(it) }
}
return TextNoteEvent.build(content, createdAt ?: TimeUtils.now()) {
@@ -54,9 +54,7 @@ data class HlsVideoPublishInput(
// (gallery thumbnails, previews) have a still to render — the .m3u8 playlist itself is a
// text manifest that can't be decoded as an image frame.
val posterUrl: String? = null,
// Blurhash + thumbhash derived once from the same poster bitmap. Threaded into every imeta
// so receiving clients can render an instant low-res placeholder before the poster JPEG
// finishes loading. Per-rendition values would be redundant — the source frame is the same.
// Identical for every rendition since the source frame is the same.
val blurhash: String? = null,
val thumbhash: String? = null,
)
@@ -73,15 +71,11 @@ sealed class HlsVideoEventTemplate {
/**
* Result of [HlsVideoEventBuilder.build]. Exposes the unsigned NIP-71 [template] plus the
* resolved coordinates the caller needs to construct a matching kind:1 sibling note: the
* orientation [kind] (34235/34236), the addressable [dTag] (replayable into an `a` tag), and
* the master [masterDimension] (largest rendition's WxH) so the kind:1 imeta carries the same
* dim as the NIP-71 master imeta.
* master [masterDimension] (largest rendition's WxH) so the kind:1 sibling imeta can carry
* the same dim as the NIP-71 master imeta.
*/
data class HlsBuiltTemplate(
val template: HlsVideoEventTemplate,
val kind: Int,
val dTag: String,
val masterDimension: DimensionTag?,
)
@@ -166,7 +160,6 @@ object HlsVideoEventBuilder {
)
}
val kind = if (isVertical) VideoVerticalEvent.KIND else VideoHorizontalEvent.KIND
return HlsBuiltTemplate(template, kind, dTag, masterDimension)
return HlsBuiltTemplate(template, masterDimension)
}
}
@@ -40,13 +40,13 @@ import androidx.media3.common.util.UnstableApi
import coil3.compose.AsyncImagePainter
import coil3.compose.SubcomposeAsyncImage
import coil3.compose.SubcomposeAsyncImageContent
import com.davotoula.lightcompressor.hls.HlsContentTypes
import com.vitorpamplona.amethyst.R
import com.vitorpamplona.amethyst.commons.icons.symbols.Icon
import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlContent
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage
import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo
import com.vitorpamplona.amethyst.commons.richtext.RichTextParser.Companion.isHlsMimeType
import com.vitorpamplona.amethyst.commons.richtext.RichTextParser.Companion.isVideoUrl
import com.vitorpamplona.amethyst.commons.richtext.toCoilModel
import com.vitorpamplona.amethyst.model.Note
@@ -69,19 +69,6 @@ import com.vitorpamplona.quartz.nip53LiveActivities.clip.LiveActivitiesClipEvent
import com.vitorpamplona.quartz.nip68Picture.PictureEvent
import com.vitorpamplona.quartz.nip71Video.VideoEvent
// Mirrors the canonical HLS-playlist mime list used in MediaItemCache.toExoPlayerMimeType.
// Kept inline rather than extracting a shared helper for one read-side caller.
private fun isHlsMimeType(mimeType: String?): Boolean =
when (mimeType?.lowercase()) {
HlsContentTypes.HLS_PLAYLIST,
"application/x-mpegurl",
"audio/x-mpegurl",
"audio/mpegurl",
-> true
else -> false
}
@Composable
fun GalleryThumbnail(
baseNote: Note,
@@ -91,14 +91,11 @@ class HlsPublishOrchestrator(
) -> HlsUploadResult<MediaUploadResult>,
private val buildUploader: (ServerName) -> HlsBlobUploader,
private val uploadMaster: suspend (HlsBlobUploader, String) -> MediaUploadResult,
// Signs and broadcasts the NIP-71 video event AND the kind:1 sibling note. The
// orchestrator hands the primary template plus a lazily-evaluated [buildSibling] closure
// that produces the kind:1 template — lazy so the orchestrator can capture the master
// metadata in scope without forcing kind:1 construction before the NIP-71 sign. Returns
// the NIP-71 event id.
// Signs and broadcasts the NIP-71 video event AND the kind:1 sibling note. Returns the
// NIP-71 event id.
private val signAndPublish: suspend (
primary: HlsVideoEventTemplate,
buildSibling: () -> EventTemplate<TextNoteEvent>,
sibling: EventTemplate<TextNoteEvent>,
) -> String,
// Generates a poster JPEG from the picked source video and uploads it via the supplied
// uploader, returning the public URL. Returns null if poster generation isn't possible
@@ -258,7 +255,7 @@ class HlsPublishOrchestrator(
thumbhash = posterResult?.thumbhash,
),
)
val buildSibling: () -> EventTemplate<TextNoteEvent> = {
val sibling =
HlsKind1SiblingBuilder.build(
title = request.title,
description = request.description,
@@ -266,11 +263,10 @@ class HlsPublishOrchestrator(
masterSha256 = masterUpload.sha256,
masterDimension = built.masterDimension,
posterUrl = posterResult?.url,
blurhashValue = posterResult?.blurhash,
thumbhashValue = posterResult?.thumbhash,
blurhash = posterResult?.blurhash,
thumbhash = posterResult?.thumbhash,
)
}
val eventId = signAndPublish(built.template, buildSibling)
val eventId = signAndPublish(built.template, sibling)
_state.value =
HlsPublishState.Success(
@@ -34,6 +34,7 @@ import com.vitorpamplona.amethyst.service.uploads.hls.HlsBlobUploader
import com.vitorpamplona.amethyst.service.uploads.hls.HlsBlobUploaderFactory
import com.vitorpamplona.amethyst.service.uploads.hls.HlsVideoEventTemplate
import com.vitorpamplona.quartz.utils.Log
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.withContext
@@ -88,7 +89,7 @@ fun createProductionHlsPublishOrchestrator(
}
}
},
signAndPublish = { template, buildSibling ->
signAndPublish = { template, sibling ->
val inner =
when (template) {
is HlsVideoEventTemplate.Horizontal -> template.template
@@ -97,16 +98,12 @@ fun createProductionHlsPublishOrchestrator(
val signed = account.signer.sign(inner)
account.sendAutomatic(signed)
// Auto-publish the kind:1 sibling note so receivers that don't speak NIP-71 still
// see a rich preview (poster + dim + blurhash/thumbhash) and can hop to the
// addressable form via the imeta + `a` tag. Publishing the sibling must not throw
// out of the orchestrator on signer failure; the NIP-71 event is already broadcast
// and surfacing as a partial success is more useful than a hard fail.
val siblingTemplate = buildSibling()
// Sibling-publish failure is a soft warning: the NIP-71 event already landed, so a
// partial success is more useful than a hard fail.
try {
val signedSibling = account.signer.sign(siblingTemplate)
val signedSibling = account.signer.sign(sibling)
account.sendAutomatic(signedSibling)
} catch (e: kotlinx.coroutines.CancellationException) {
} catch (e: CancellationException) {
throw e
} catch (e: Throwable) {
Log.w(TAG) { "kind:1 sibling sign/publish failed: ${e.message}" }
@@ -141,10 +138,14 @@ private suspend fun generateAndUploadPoster(
// JPEG receiving clients will fetch via the imeta `image` URL.
val posterBytes = posterFile.readBytes()
val hashes =
runCatching {
try {
PreviewMetadataCalculator.computeFromBytes(posterBytes, POSTER_CONTENT_TYPE, null)
}.onFailure { Log.w(TAG) { "uploadPoster: blurhash/thumbhash compute failed: ${it.message}" } }
.getOrNull()
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
Log.w(TAG) { "uploadPoster: blurhash/thumbhash compute failed: ${e.message}" }
null
}
val result = uploader.upload(posterFile, POSTER_CONTENT_TYPE) { _, _ -> }
val url = result.url ?: return null
HlsPosterUpload(
@@ -185,6 +186,8 @@ private suspend fun extractPosterToTempFile(
}
throw e
}
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
Log.w(TAG) { "extractPosterToTempFile: failed for $uri${e.message}" }
null
@@ -0,0 +1,150 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.amethyst.service.uploads.hls
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
import com.vitorpamplona.quartz.nip94FileMetadata.tags.DimensionTag
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
class HlsKind1SiblingBuilderTest {
private val masterUrl = "https://cdn.test/master.m3u8"
private val masterSha = "ffeeddccbbaa00112233445566778899aabbccddeeff00112233445566778899"
private val posterUrl = "https://cdn.test/poster.jpg"
private val blurhash = "LFE.@D9F01_2~q%2tRj["
private val thumbhash = "wJlGAA"
private val masterDim = DimensionTag(2160, 3840)
private fun build(
title: String = "My HD video",
description: String = "A clip",
masterSha256: String? = masterSha,
masterDimension: DimensionTag? = masterDim,
posterUrlArg: String? = posterUrl,
blurhashArg: String? = blurhash,
thumbhashArg: String? = thumbhash,
createdAt: Long? = 1_700_000_000L,
) = HlsKind1SiblingBuilder.build(
title = title,
description = description,
masterUrl = masterUrl,
masterSha256 = masterSha256,
masterDimension = masterDimension,
posterUrl = posterUrlArg,
blurhash = blurhashArg,
thumbhash = thumbhashArg,
createdAt = createdAt,
)
private fun Array<Array<String>>.findTag(name: String): Array<String>? = firstOrNull { it.isNotEmpty() && it[0] == name }
@Test
fun kindIs1() {
val template = build()
assertEquals(TextNoteEvent.KIND, template.kind)
}
@Test
fun contentJoinsTitleDescriptionMasterUrlOnDoubleNewline() {
val template = build(title = "T", description = "D")
assertEquals("T\n\nD\n\n$masterUrl", template.content)
}
@Test
fun blankFieldsAreOmittedFromContent() {
val template = build(title = "", description = " ")
assertEquals(masterUrl, template.content)
}
@Test
fun fieldsAreTrimmed() {
val template = build(title = " T ", description = "\nD\n")
assertEquals("T\n\nD\n\n$masterUrl", template.content)
}
@Test
fun masterImetaCarriesAllVisualFields() {
val template = build()
val imeta = template.tags.findTag("imeta")
assertNotNull(imeta)
val flat = imeta!!.joinToString("|")
assertTrue("url: $flat", flat.contains("url $masterUrl"))
assertTrue("mime: $flat", flat.contains("m application/vnd.apple.mpegurl"))
assertTrue("hash: $flat", flat.contains("x $masterSha"))
assertTrue("dim: $flat", flat.contains("dim 2160x3840"))
assertTrue("image: $flat", flat.contains("image $posterUrl"))
assertTrue("blurhash: $flat", flat.contains("blurhash $blurhash"))
assertTrue("thumbhash: $flat", flat.contains("thumbhash $thumbhash"))
}
@Test
fun nullVisualFieldsAreOmittedFromImeta() {
val template =
build(
masterSha256 = null,
masterDimension = null,
posterUrlArg = null,
blurhashArg = null,
thumbhashArg = null,
)
val imeta = template.tags.findTag("imeta")!!
val flat = imeta.joinToString("|")
assertTrue("url is required: $flat", flat.contains("url $masterUrl"))
assertTrue("mime is always present: $flat", flat.contains("m application/vnd.apple.mpegurl"))
assertFalse("no hash: $flat", flat.contains("x "))
assertFalse("no dim: $flat", flat.contains("dim "))
assertFalse("no image: $flat", flat.contains("image "))
assertFalse("no blurhash: $flat", flat.contains("blurhash "))
assertFalse("no thumbhash: $flat", flat.contains("thumbhash "))
}
@Test
fun rTagCarriesMasterUrl() {
val template = build()
val ref = template.tags.findTag("r")
assertNotNull(ref)
assertEquals(masterUrl, ref!![1])
}
@Test
fun noATagIsEmitted() {
val template = build()
assertNull(template.tags.findTag("a"))
}
@Test
fun createdAtUsesProvidedValue() {
val template = build(createdAt = 12_345L)
assertEquals(12_345L, template.createdAt)
}
@Test
fun createdAtDefaultsToNowWhenNull() {
val before = System.currentTimeMillis() / 1_000
val template = build(createdAt = null)
val after = System.currentTimeMillis() / 1_000
assertTrue(template.createdAt in before..after)
}
}
@@ -34,6 +34,8 @@ import com.vitorpamplona.amethyst.ui.screen.loggedIn.video.hls.HlsPosterUpload
import com.vitorpamplona.amethyst.ui.screen.loggedIn.video.hls.HlsPublishOrchestrator
import com.vitorpamplona.amethyst.ui.screen.loggedIn.video.hls.HlsPublishRequest
import com.vitorpamplona.amethyst.ui.screen.loggedIn.video.hls.HlsPublishState
import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate
import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.runBlocking
import org.junit.After
@@ -377,6 +379,7 @@ class HlsPublishOrchestratorTest {
@Test
fun posterUrlFromUploadPosterClosureLandsOnEveryImeta() {
val captured = mutableListOf<HlsVideoEventTemplate>()
val capturedSiblings = mutableListOf<EventTemplate<TextNoteEvent>>()
val canned = CannedUploader()
val orchestrator =
HlsPublishOrchestrator(
@@ -384,8 +387,9 @@ class HlsPublishOrchestratorTest {
runUpload = fakeRunUpload(),
buildUploader = { canned },
uploadMaster = fakeUploadMaster(canned),
signAndPublish = { tpl, _ ->
signAndPublish = { tpl, sibling ->
captured += tpl
capturedSiblings += sibling
"event-id"
},
uploadPoster = { _ -> HlsPosterUpload("https://cdn.test/poster.jpg") },
@@ -400,6 +404,13 @@ class HlsPublishOrchestratorTest {
val flat = imeta.joinToString("|")
assertTrue("imeta missing poster: $flat", flat.contains("image https://cdn.test/poster.jpg"))
}
// Sibling kind:1 carries the same poster URL on its single imeta.
val sibling = capturedSiblings.single()
val siblingImeta = sibling.tags.firstOrNull { it.isNotEmpty() && it[0] == "imeta" }
assertNotNull("sibling missing imeta", siblingImeta)
val siblingFlat = siblingImeta!!.joinToString("|")
assertTrue("sibling imeta missing poster: $siblingFlat", siblingFlat.contains("image https://cdn.test/poster.jpg"))
}
@Test
@@ -31,6 +31,7 @@ import com.vitorpamplona.quartz.nip92IMeta.imetasByUrl
import com.vitorpamplona.quartz.nip94FileMetadata.tags.BlurhashTag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.DimensionTag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.HashSha256Tag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.ImageTag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.MimeTypeTag
import com.vitorpamplona.quartz.nip94FileMetadata.tags.ThumbhashTag
import com.vitorpamplona.quartz.utils.Log
@@ -64,7 +65,12 @@ class RichTextParser {
if (contentType != null) {
isImage = contentType.startsWith("image/")
isVideo = contentType.startsWith("video/") || contentType.startsWith("audio/")
// HLS playlists are advertised with a non-`video/*` MIME (`application/vnd.apple.mpegurl`
// and three legacy aliases). Without these, an imeta-described `.m3u8` falls into the
// null bucket below and the renderer drops back to a plain hyperlink — even though
// the matching extension would have routed it to MediaUrlVideo. Mirror the canonical
// list used by MediaItemCache.toExoPlayerMimeType / GalleryThumb.isHlsMimeType.
isVideo = contentType.startsWith("video/") || contentType.startsWith("audio/") || isHlsMimeType(contentType)
isPdf = contentType.startsWith("application/pdf")
} else if (fullUrl.startsWith("data:")) {
isImage = fullUrl.startsWith("data:image/")
@@ -99,6 +105,10 @@ class RichTextParser {
dim = frags[DimensionTag.TAG_NAME]?.let { DimensionTag.parse(it) } ?: tags[DimensionTag.TAG_NAME]?.firstOrNull()?.let { DimensionTag.parse(it) },
contentWarning = frags[ContentWarningTag.TAG_NAME] ?: tags[ContentWarningTag.TAG_NAME]?.firstOrNull(),
uri = callbackUri,
// Poster URL from the imeta's `image` property — downstream gallery-add reads
// this as the entry's `image` tag so the gallery thumbnail can render the
// poster JPEG instead of falling back to the blurhash placeholder.
artworkUri = frags[ImageTag.TAG_NAME] ?: tags[ImageTag.TAG_NAME]?.firstOrNull(),
mimeType = contentType,
thumbhash = frags[ThumbhashTag.TAG_NAME] ?: tags[ThumbhashTag.TAG_NAME]?.firstOrNull(),
authorPubKey = authorPubKey,
@@ -457,6 +467,17 @@ class RichTextParser {
return videoExtensions.any { removedParamsFromUrl.endsWith(it) }
}
// Mirrors the canonical HLS-playlist MIME list also kept in MediaItemCache.toExoPlayerMimeType.
// Called per URL during feed render — uses `equals(ignoreCase)` instead of `lowercase()` to
// avoid a per-call String allocation on the common non-HLS path.
fun isHlsMimeType(mimeType: String?): Boolean {
if (mimeType == null) return false
return mimeType.equals("application/vnd.apple.mpegurl", ignoreCase = true) ||
mimeType.equals("application/x-mpegurl", ignoreCase = true) ||
mimeType.equals("audio/x-mpegurl", ignoreCase = true) ||
mimeType.equals("audio/mpegurl", ignoreCase = true)
}
fun isPdfUrl(url: String): Boolean {
val removedParamsFromUrl = removeQueryParamsForExtensionComparison(url)
return pdfExtensions.any { removedParamsFromUrl.endsWith(it) }