Switches to our own version of the Url Detector

2026-03-06 10:58:59 -05:00
parent 13f401df8c
commit f4d401bc56
32 changed files with 3794 additions and 156 deletions
@@ -20,8 +20,6 @@
 */
 package com.vitorpamplona.amethyst.commons.richtext

-import com.linkedin.urls.detection.UrlDetector
-import com.linkedin.urls.detection.UrlDetectorOptions
 import com.vitorpamplona.amethyst.commons.emojicoder.EmojiCoder
 import com.vitorpamplona.amethyst.commons.model.ImmutableListOfLists
 import com.vitorpamplona.quartz.experimental.inlineMetadata.Nip54InlineMetadata
@@ -39,7 +37,6 @@ import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.toImmutableList
 import kotlinx.collections.immutable.toImmutableMap
-import kotlinx.collections.immutable.toImmutableSet
 import kotlinx.collections.immutable.toPersistentList
 import java.net.MalformedURLException
 import java.net.URISyntaxException
@@ -101,27 +98,43 @@ class RichTextParser {
        }
    }

-    fun parseValidUrls(content: String): LinkedHashSet<String> {
-        val urls = UrlDetector(content, UrlDetectorOptions.Default).detect()
+    fun fixMissingSpaces(
+        input: String,
+        urlList: Set<String>,
+    ): String {
+        if (urlList.isEmpty()) return input

-        return urls.mapNotNullTo(LinkedHashSet(urls.size)) {
-            if (it.originalUrl.contains("@")) {
-                if (Patterns.EMAIL_ADDRESS.matches(it.originalUrl)) {
-                    null
-                } else {
-                    it.originalUrl
-                }
-            } else if (isNumber(it.originalUrl)) {
-                null // avoids urls that look like 123.22
-            } else if (it.originalUrl.contains("。")) {
-                null // avoids Japanese characters as fake urls
-            } else {
-                if (HTTPRegex.matches(it.originalUrl)) {
-                    it.originalUrl
-                } else {
-                    null
-                }
+        // Escape and join words: (word1|word2)
+        val wordsPattern = urlList.sortedByDescending { it.length }.joinToString("|") { Regex.escape(it) }
+
+        // Regex breakdown:
+        // ([^ ])?          -> Group 1: Optional character that is NOT a space or new line (Prefix)
+        // ($wordsPattern)  -> Group 2: One of your target words
+        // ([^ ])?          -> Group 3: Optional character that is NOT a space or new line (Suffix)
+        val regex = Regex("([^ \n])?($wordsPattern)([^ \n])?")
+
+        return regex.replace(input) { match ->
+            val prefix = match.groups[1]?.value ?: ""
+            val word = match.groups[2]?.value ?: ""
+            val suffix = match.groups[3]?.value ?: ""
+
+            val result = StringBuilder()
+
+            // Add prefix + space if the prefix exists
+            if (prefix.isNotEmpty()) {
+                result.append(prefix)
+                result.append(" ")
            }
+
+            result.append(word)
+
+            // Add space + suffix if the suffix exists
+            if (suffix.isNotEmpty()) {
+                result.append(" ")
+                result.append(suffix)
+            }
+
+            result.toString()
        }
    }

@@ -131,30 +144,41 @@ class RichTextParser {
        callbackUri: String?,
    ): RichTextViewerState {
        val imetas = tags.lists.imetasByUrl()
-        val urlSet = parseValidUrls(content)
+        val urlSet = UrlParser().parseValidUrls(content)

-        val imagesForPager =
-            urlSet.mapNotNull { fullUrl -> createMediaContent(fullUrl, imetas, content, callbackUri) }.associateBy { it.url }
+        val mediaContents =
+            urlSet.withScheme.mapNotNull { fullUrl ->
+                createMediaContent(fullUrl, imetas, content, callbackUri)
+            } +
+                urlSet.withoutScheme.mapNotNull { fullUrl ->
+                    createMediaContent(fullUrl, imetas, content, callbackUri)
+                }

-        val imageUrls = imagesForPager.filterValues { it is MediaUrlImage }.keys
-        val videoUrls = imagesForPager.filterValues { it is MediaUrlVideo }.keys
+        val mediaForPager = mediaContents.associateBy { it.url }
+
+        val imageUrls = mediaForPager.filterValues { it is MediaUrlImage }.keys
+        val videoUrls = mediaForPager.filterValues { it is MediaUrlVideo }.keys

        val emojiMap = CustomEmoji.createEmojiMap(tags.lists)

-        val segments = findTextSegments(content, imageUrls, videoUrls, urlSet, emojiMap, tags)
+        val allUrls = urlSet.withScheme + urlSet.withoutScheme + urlSet.emails

-        val base64Images = segments.map { it.words.filterIsInstance<Base64Segment>() }.flatten()
+        val newContent = fixMissingSpaces(content, allUrls)

-        val imagesForPagerWithBase64 =
-            imagesForPager +
+        val segments = findTextSegments(newContent, imageUrls, videoUrls, urlSet, emojiMap, tags)
+
+        val base64Images = segments.flatMap { it.words.filterIsInstance<Base64Segment>() }
+
+        val mediaForPagerWithBase64 =
+            mediaForPager +
                base64Images
                    .mapNotNull { createMediaContent(it.segmentText, emptyMap(), content, callbackUri) }
                    .associateBy { it.url }

        return RichTextViewerState(
-            urlSet.toImmutableSet(),
-            imagesForPagerWithBase64.toImmutableMap(),
-            imagesForPagerWithBase64.values.toImmutableList(),
+            urlSet,
+            mediaForPagerWithBase64.toImmutableMap(),
+            mediaForPagerWithBase64.values.toImmutableList(),
            emojiMap.toImmutableMap(),
            segments,
            tags,
@@ -165,7 +189,7 @@ class RichTextParser {
        content: String,
        images: Set<String>,
        videos: Set<String>,
-        urls: Set<String>,
+        urls: Urls,
        emojis: Map<String, String>,
        tags: ImmutableListOfLists<String>,
    ): ImmutableList<ParagraphState> {
@@ -175,18 +199,14 @@ class RichTextParser {
        lines.forEach { paragraph ->
            val isRTL = isArabic(paragraph)

-            val wordList = paragraph.trimEnd().split(wordBoundaryRegex).filter { it.isNotEmpty() }
+            val wordList = paragraph.trimEnd().split(' ')

-            if (wordList.isEmpty()) {
-                paragraphSegments.add(ParagraphState(persistentListOf(RegularTextSegment("")), isRTL))
-            } else {
-                val segments = ArrayList<Segment>(wordList.size)
-                wordList.forEach { word ->
-                    segments.add(wordIdentifier(word, images, videos, urls, emojis, tags))
-                }
-
-                paragraphSegments.add(ParagraphState(segments.toPersistentList(), isRTL))
+            val segments = ArrayList<Segment>(wordList.size)
+            wordList.forEach { word ->
+                segments.add(wordIdentifier(word, images, videos, urls, emojis, tags))
            }
+
+            paragraphSegments.add(ParagraphState(segments.toPersistentList(), isRTL))
        }

        val segmentsWithGalleries = GalleryParser().processParagraphs(paragraphSegments)
@@ -204,8 +224,6 @@ class RichTextParser {
            }.toImmutableList()
    }

-    private fun isNumber(word: String) = numberPattern.matches(word)
-
    private fun isPhoneNumberChar(c: Char): Boolean =
        when (c) {
            in '0'..'9' -> true
@@ -236,7 +254,7 @@ class RichTextParser {
        word: String,
        images: Set<String>,
        videos: Set<String>,
-        urls: Set<String>,
+        urls: Urls,
        emojis: Map<String, String>,
        tags: ImmutableListOfLists<String>,
    ): Segment {
@@ -246,13 +264,29 @@ class RichTextParser {
            if (Patterns.BASE64_IMAGE.matches(word)) return Base64Segment(word)
        }

-        if (images.contains(word)) return ImageSegment(word)
+        if (images.contains(word)) {
+            return if (urls.withoutScheme.contains(word)) {
+                ImageSegment("https://$word")
+            } else {
+                ImageSegment(word)
+            }
+        }

-        if (videos.contains(word)) return VideoSegment(word)
+        if (videos.contains(word)) {
+            return if (urls.withoutScheme.contains(word)) {
+                VideoSegment("https://$word")
+            } else {
+                VideoSegment(word)
+            }
+        }

        if (word.startsWith("ws://") || word.startsWith("wss://")) return RelayUrlSegment(word)

-        if (urls.contains(word)) return LinkSegment(word)
+        if (urls.withoutScheme.contains(word)) {
+            return LinkSegment("https://$word")
+        } else if (urls.withScheme.contains(word)) {
+            return LinkSegment(word)
+        }

        if (CustomEmoji.fastMightContainEmoji(word, emojis) && emojis.any { word.contains(it.key) }) return EmojiSegment(word)

@@ -262,13 +296,11 @@ class RichTextParser {

        if (word.startsWith("cashuA", true) || word.startsWith("cashuB", true)) return CashuSegment(word)

-        if (word.startsWith("#")) return parseHash(word, tags)
+        if (word.startsWith('#')) return parseHash(word, tags)

        if (EmojiCoder.isCoded(word)) return SecretEmoji(word)

-        if (word.contains("@")) {
-            if (Patterns.EMAIL_ADDRESS.matches(word)) return EmailSegment(word)
-        }
+        if (urls.emails.contains(word)) return EmailSegment(word)

        if (startsWithNIP19Scheme(word)) return BechSegment(word)

@@ -276,18 +308,6 @@ class RichTextParser {
            if (Patterns.PHONE.matches(word)) return PhoneSegment(word)
        }

-        val indexOfPeriod = word.indexOf(".")
-        if (indexOfPeriod > 0 && indexOfPeriod < word.length - 1) { // periods cannot be the last one
-            val schemelessMatcher = noProtocolUrlValidator.find(word)
-            if (schemelessMatcher != null) {
-                val url = schemelessMatcher.groups[1]?.value // url
-                val additionalChars = schemelessMatcher.groups[4]?.value?.ifEmpty { null } // additional chars
-                if (additionalUrlSchema.find(word) != null && url != null) {
-                    return SchemelessUrlSegment(word, url, additionalChars)
-                }
-            }
-        }
-
        return RegularTextSegment(word)
    }

@@ -343,7 +363,7 @@ class RichTextParser {

        val noProtocolUrlValidator =
            Regex(
-                "(([a-zA-Z0-9_-]+\\.)*[a-zA-Z][a-zA-Z0-9_-]+[\\.\\:][a-zA-Z0-9_]+([\\/ \\?\\=\\&\\#\\.]?[a-zA-Z0-9_-]+)*\\/?)(.*)",
+                "(([a-zA-Z0-9_-]+@)?([a-zA-Z0-9_-]+\\.)*[a-zA-Z0-9_-]+[\\.\\:][a-zA-Z0-9_]+([\\/ \\?\\=\\&\\#\\.]?[a-zA-Z0-9_-]+)*\\/?)(.*)",
            )

        // Splits at spaces AND at ASCII/multibyte character boundaries
@@ -24,11 +24,10 @@ import androidx.compose.runtime.Immutable
 import com.vitorpamplona.amethyst.commons.model.ImmutableListOfLists
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.ImmutableMap
-import kotlinx.collections.immutable.ImmutableSet

@Immutable
 class RichTextViewerState(
-    val urlSet: ImmutableSet<String>,
+    val urlSet: Urls,
    val imagesForPager: ImmutableMap<String, MediaUrlContent>,
    val imageList: ImmutableList<MediaUrlContent>,
    val customEmoji: ImmutableMap<String, String>,
@@ -139,13 +138,6 @@ class HashTagSegment(
    val extras: String?,
 ) : Segment(segment)

-@Immutable
-class SchemelessUrlSegment(
-    segment: String,
-    val url: String,
-    val extras: String?,
-) : Segment(segment)
-
@Immutable
 class RelayUrlSegment(
    segment: String,
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.commons.richtext
+
+import com.vitorpamplona.amethyst.commons.richtext.RichTextParser.Companion.noProtocolUrlValidator
+import com.vitorpamplona.quartz.utils.urldetector.Url
+import com.vitorpamplona.quartz.utils.urldetector.detection.UrlDetector
+
+class Urls(
+    val withScheme: Set<String> = emptySet(),
+    val withoutScheme: Set<String> = emptySet(),
+    val emails: Set<String> = emptySet(),
+)
+
+class UrlParser {
+    fun Char.isAsciiLetter(): Boolean = (this in 'a'..'z' || this in 'A'..'Z')
+
+    fun Url.isValidTopLevelDomain(): Boolean {
+        /*
+        According to the TLD Applicant Guidebook published June 2012, ICANN does not allow numbers in TLDs.
+         */
+        val startOfTopDomain = host.lastIndexOf('.') + 1
+        return if (startOfTopDomain < host.length) {
+            val topLevelDomain = host.substring(startOfTopDomain)
+            topLevelDomain.isNotEmpty() && topLevelDomain[0].isAsciiLetter()
+        } else {
+            false
+        }
+    }
+
+    fun Url.wroteWithSchema(): Boolean = originalUrl.startsWith(scheme)
+
+    fun Url.isEmail(): Boolean = originalUrl.contains('@') && path == "/" && query.isEmpty() && fragment.isEmpty()
+
+    fun Char.isValidLastHostnameChar(): Boolean = (this in 'a'..'z' || this in 'A'..'Z' || this in '0'..'9')
+
+    fun Url.isValidLastHostnameChar(): Boolean = host[host.length - 1].isValidLastHostnameChar()
+
+    fun Url.endsWithHost(): Boolean = originalUrl.endsWith(host)
+
+    val notAHostNameChar = "[^a-zA-Z0-9.-]".toRegex()
+
+    fun parseValidUrls(content: String): Urls {
+        val urls = UrlDetector(content).detect()
+
+        val completeUrls = mutableSetOf<String>()
+        val urlsWithoutScheme = mutableSetOf<String>()
+        val emails = mutableSetOf<String>()
+
+        println("AABBBCC parseValidUrls ${urls.size}")
+
+        urls.forEach {
+            println("AABBBCC Testing ${it.originalUrl}")
+            if (it.isValidTopLevelDomain()) {
+                if (it.wroteWithSchema()) {
+                    if (it.isValidLastHostnameChar()) {
+                        completeUrls.add(it.originalUrl)
+                    } else if (it.endsWithHost()) {
+                        val match = notAHostNameChar.find(it.host)
+                        if (match != null) {
+                            completeUrls.add(it.originalUrl.substring(0, (it.originalUrl.length - it.host.length) + match.range.first))
+                        } else {
+                            completeUrls.add(it.originalUrl)
+                        }
+                    }
+                } else {
+                    // emails are understood as urls from the detector.
+                    if (it.isEmail()) {
+                        Patterns.EMAIL_ADDRESS.findAll(it.originalUrl).forEach {
+                            emails.add(it.value)
+                        }
+                    } else {
+                        noProtocolUrlValidator.findAll(it.originalUrl).forEach { components ->
+                            val url = components.groups[1]?.value
+                            if (url != null) {
+                                urlsWithoutScheme.add(url)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return Urls(
+            withScheme = completeUrls,
+            withoutScheme = urlsWithoutScheme,
+            emails = emails,
+        )
+    }
+}
@@ -1,3 +1,23 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
 package com.vitorpamplona.amethyst.commons.richtext

 import kotlin.test.Test
@@ -55,7 +55,55 @@ class RichTextParserMultibyteTest {
        // user@example.com should not be in urlSet
        assertTrue(
            "user@example.com should not be in urlSet",
-            !state.urlSet.contains("user@example.com"),
+            !state.urlSet.withScheme.contains("user@example.com") && !state.urlSet.withoutScheme.contains("user@example.com"),
+        )
+    }
+
+    @Test
+    fun testHttpWithoutSpaces() {
+        // Multibyte characters around an email address should not produce URL/Link segments
+        val text =
+            "Vitor, vocêhttp://test.com? \uD83E\uDD7A"
+
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)
+
+        assertEquals(
+            "Vitor, você http://test.com? \uD83E\uDD7A",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
+        )
+    }
+
+    @Test
+    fun testHttpWithoutSpacesJapan() {
+        // Multibyte characters around an email address should not produce URL/Link segments
+        val text =
+            "Vitor, vocêhttp://test.comほげほげ"
+
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)
+
+        assertEquals(
+            "Vitor, você http://test.com ほげほげ",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
+        )
+    }
+
+    @Test
+    fun testHttpWithoutSpacesJapan2() {
+        // Multibyte characters around an email address should not produce URL/Link segments
+        val text =
+            "Vitor, vocêhttp://test.com。ほげほげ"
+
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)
+
+        assertEquals(
+            "Vitor, você http://test.com 。ほげほげ",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
        )
    }

@@ -65,11 +113,29 @@ class RichTextParserMultibyteTest {
        val text =
            "Vitor, você tem como colocar alguma forma de aviso se o link vai carregar uma imagem ou um vídeo? \uD83E\uDD7A"

-        val regex = Regex("(?<=[\\u0000-\\u00FF])(?=[\\u0100-\\uFFFF])|(?<=[\\u0100-\\uFFFF])(?=[\\u0000-\\u00FF])| +")
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)

        assertEquals(
-            "Vitor,-você-tem-como-colocar-alguma-forma-de-aviso-se-o-link-vai-carregar-uma-imagem-ou-um-vídeo?-\uD83E\uDD7A",
-            text.split(regex).joinToString("-"),
+            "Vitor, você tem como colocar alguma forma de aviso se o link vai carregar uma imagem ou um vídeo? \uD83E\uDD7A",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
+        )
+    }
+
+    @Test
+    fun testFullTextWithMultibyteAndQuotes() {
+        // Multibyte characters around an email address should not produce URL/Link segments
+        val text =
+            "I’ve been thinking lately about how I believe there will more than likely be models unattainable by most. Think Bloomberg Terminal. Where their cost of tokens is too high for the average lay person, but their level of “cognition” is unmatched by anything else. I’m sure there will even be many closed models that are invite only. Crazy times ahead."
+
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)
+
+        assertEquals(
+            "I’ve been thinking lately about how I believe there will more than likely be models unattainable by most. Think Bloomberg Terminal. Where their cost of tokens is too high for the average lay person, but their level of “cognition” is unmatched by anything else. I’m sure there will even be many closed models that are invite only. Crazy times ahead.",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
        )
    }

@@ -91,9 +157,9 @@ class RichTextParserMultibyteTest {
        val state = RichTextParser().parseText(text, EmptyTagList, null)
        val allSegments = state.paragraphs.flatMap { it.words }

-        val urlSegments = allSegments.filterIsInstance<SchemelessUrlSegment>()
+        val urlSegments = allSegments.filterIsInstance<LinkSegment>()
        assertTrue("Should have SchemelessUrlSegment", urlSegments.isNotEmpty())
-        assertTrue("URL should be example.com", urlSegments.any { it.url == "example.com" })
+        assertTrue("URL should be example.com", urlSegments.any { it.segmentText == "https://example.com" })

        val textSegments = allSegments.filterIsInstance<RegularTextSegment>()
        assertTrue("Should have prefix ああ", textSegments.any { it.segmentText == "ああ" })
@@ -106,9 +172,9 @@ class RichTextParserMultibyteTest {
        val state = RichTextParser().parseText(text, EmptyTagList, null)
        val allSegments = state.paragraphs.flatMap { it.words }

-        val urlSegments = allSegments.filterIsInstance<SchemelessUrlSegment>()
+        val urlSegments = allSegments.filterIsInstance<LinkSegment>()
        assertTrue("Should have SchemelessUrlSegment", urlSegments.isNotEmpty())
-        assertTrue("URL should be example.com", urlSegments.any { it.url == "example.com" })
+        assertTrue("URL should be example.com", urlSegments.any { it.segmentText == "https://example.com" })

        val textSegments = allSegments.filterIsInstance<RegularTextSegment>()
        assertTrue("Should have suffix ああ", textSegments.any { it.segmentText == "ああ" })
@@ -0,0 +1,290 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.commons.richtext
+
+import kotlin.test.Ignore
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class UrlParserTest {
+    val parser = UrlParser()
+
+    fun test(
+        text: String,
+        expected: Urls,
+    ) {
+        val urlSet = parser.parseValidUrls(text)
+        assertEquals(expected.withScheme, urlSet.withScheme)
+        assertEquals(expected.withoutScheme, urlSet.withoutScheme)
+        assertEquals(expected.emails, urlSet.emails)
+    }
+
+    @Test
+    fun testSimpleText() =
+        test(
+            "test. com",
+            Urls(),
+        )
+
+    @Test
+    fun testBasicUrl() =
+        test(
+            "http://test.com",
+            Urls(withScheme = setOf("http://test.com")),
+        )
+
+    @Test
+    fun testNoSchemaUrl() =
+        test(
+            "test.com",
+            Urls(withoutScheme = setOf("test.com")),
+        )
+
+    @Test
+    fun testNoSchemaUrlPrefixMultibyte() =
+        test(
+            "ほtest.com",
+            Urls(withoutScheme = setOf("test.com")),
+        )
+
+    @Test
+    fun testNoSchemaUrlSuffixMultibyte() =
+        test(
+            "test.comほ",
+            Urls(withoutScheme = setOf("test.com")),
+        )
+
+    @Test
+    fun testNoSchemaUrlWithParams() =
+        test(
+            "test.com/some/me/hey?param=value#some=value",
+            Urls(withoutScheme = setOf("test.com/some/me/hey?param=value#some=value")),
+        )
+
+    @Test
+    fun testNoSchemaUrlWithParamsWithOtherWords() =
+        test(
+            "Hi there, check my website test.com/some/me/hey?param=value#some=value .",
+            Urls(withoutScheme = setOf("test.com/some/me/hey?param=value#some=value")),
+        )
+
+    @Test
+    fun testBasicUrlWithoutSpaceBefore() =
+        test(
+            "ahttp://test.com",
+            Urls(withScheme = setOf("http://test.com")),
+        )
+
+    @Test
+    fun testBasicUrlWithoutSpaceBeforeMultiByte() =
+        test(
+            "ほhttp://test.com",
+            Urls(withScheme = setOf("http://test.com")),
+        )
+
+    @Test
+    fun testBasicUrlWithoutSpaceAfter() =
+        test(
+            "http://test.comほ",
+            Urls(withScheme = setOf("http://test.com")),
+        )
+
+    @Test
+    fun testBasicUrlWithMultibytePath() =
+        test(
+            "http://test.com/ほ",
+            Urls(withScheme = setOf("http://test.com/ほ")),
+        )
+
+    @Test
+    fun testBasicUrls() =
+        test(
+            "http://test.com http://test2.com",
+            Urls(withScheme = setOf("http://test.com", "http://test2.com")),
+        )
+
+    @Test
+    fun testEmail() =
+        test(
+            "vitor@vitorpamplona.com",
+            Urls(emails = setOf("vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testEmailWithMultibytePrefix() =
+        test(
+            "ほvitor@vitorpamplona.com",
+            Urls(emails = setOf("vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testEmailWithMultibyteSuffix() =
+        test(
+            "vitor@vitorpamplona.comほ",
+            Urls(emails = setOf("vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testEmailWithMultibyteBoth() =
+        test(
+            "ほvitor@vitorpamplona.comほ",
+            Urls(emails = setOf("vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testUrlWithUserAndMultibyteSuffix() =
+        test(
+            "http://vitor@vitorpamplona.comほ",
+            Urls(withScheme = setOf("http://vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testUrlWithUserAndMultibytePrefix() =
+        test(
+            "ほhttp://vitor@vitorpamplona.com",
+            Urls(withScheme = setOf("http://vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testNostrUrls() =
+        test(
+            "nostr:npub1aabbcc",
+            Urls(),
+        )
+
+    @Test
+    fun testUrlsWithUsernameAndPath() =
+        test(
+            "miceliomad@miceliomad.github.io/nostr/",
+            Urls(withoutScheme = setOf("miceliomad@miceliomad.github.io/nostr/")),
+        )
+
+    @Test
+    fun testUrlsWithQuery() =
+        test(
+            " universe.nostrich.land?lang=zh ",
+            Urls(withoutScheme = setOf("universe.nostrich.land?lang=zh")),
+        )
+
+    @Test
+    fun testUrlsWithSchemaPathAndQuery() =
+        test(
+            "https://miceliomad.github.io/nostr/test?me=you",
+            Urls(withScheme = setOf("https://miceliomad.github.io/nostr/test?me=you")),
+        )
+
+    @Test
+    fun testUrlsWithPathAndQuery() =
+        test(
+            "miceliomad.github.io/nostr/test?me=you",
+            Urls(withoutScheme = setOf("miceliomad.github.io/nostr/test?me=you")),
+        )
+
+    @Test
+    fun testAvifFileNameComplete() =
+        test(
+            "https://bae.st/media/66b08dde784287ed8f92c455bc62076a04671ccb44097550626a532185a5d3ed.avif?name=81ca16-b665-4f57-80cb-11a58461fb61.avif",
+            Urls(withScheme = setOf("https://bae.st/media/66b08dde784287ed8f92c455bc62076a04671ccb44097550626a532185a5d3ed.avif?name=81ca16-b665-4f57-80cb-11a58461fb61.avif")),
+        )
+
+    @Test
+    fun testAvifFileName() =
+        test(
+            "81ca16-b665-4f57-80cb-11a58461fb61.avif",
+            Urls(withoutScheme = setOf("81ca16-b665-4f57-80cb-11a58461fb61.avif")),
+        )
+
+    @Test
+    fun testMultiLine() =
+        test(
+            """
+            22.8K (3.2%) nos.lol
+            22.7K (3.1%) universe.nostrich.land?lang=zh
+            22.5K (3.1%) universe.nostrich.land?lang=en
+            """.trimIndent(),
+            Urls(withoutScheme = setOf("nos.lol", "universe.nostrich.land?lang=zh", "universe.nostrich.land?lang=en")),
+        )
+
+    @Test
+    fun testEmailWithDashes() =
+        test(
+            "freeverification@Nostr-Check.com",
+            Urls(emails = setOf("freeverification@Nostr-Check.com")),
+        )
+
+    @Test
+    fun testEmailWithManyDashes() =
+        test(
+            "free-veri-fica-tion@No-str-Ch-eck.com",
+            Urls(emails = setOf("free-veri-fica-tion@No-str-Ch-eck.com")),
+        )
+
+    @Test
+    fun testEmailWithUnderscore() =
+        test(
+            "vi_t_or@vitorpamplona.com",
+            Urls(emails = setOf("vi_t_or@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testEmailsWithPeriod() =
+        test(
+            "john.smith@gmail.com",
+            Urls(emails = setOf("john.smith@gmail.com")),
+        )
+
+    @Test
+    fun testStrangeError() =
+        test(
+            "neomobius_at_mstdn.jp@mostr.pub",
+            Urls(emails = setOf("neomobius_at_mstdn.jp@mostr.pub")),
+        )
+
+    @Test
+    @Ignore("We need to make this work")
+    fun testRelayUrl() =
+        test(
+            "wss://test.com",
+            Urls(withScheme = setOf("wss://test.com")),
+        )
+
+    @Test
+    @Ignore("We need to make this work")
+    fun testBech12() =
+        test(
+            "nostr:npub1aabbcc",
+            Urls(withScheme = setOf("wss://test.com")),
+        )
+
+    @Test
+    fun testJapaneseUrls() =
+        test(
+            "我进入你的主页很卡顿，也许是你的关注人数或者其他数据太多了，其他人主页没有这么卡顿。来自amethyst客户端",
+            Urls(withScheme = emptySet()),
+        )
+
+    @Test
+    fun testHour() =
+        test(
+            "10.00hr,",
+            Urls(withScheme = emptySet()),
+        )
+}