Switches to our own version of the Url Detector

2026-03-06 10:58:59 -05:00
parent 13f401df8c
commit f4d401bc56
32 changed files with 3794 additions and 156 deletions
@@ -1,3 +1,23 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
 package com.vitorpamplona.amethyst.commons.richtext

 import kotlin.test.Test
@@ -55,7 +55,55 @@ class RichTextParserMultibyteTest {
        // user@example.com should not be in urlSet
        assertTrue(
            "user@example.com should not be in urlSet",
-            !state.urlSet.contains("user@example.com"),
+            !state.urlSet.withScheme.contains("user@example.com") && !state.urlSet.withoutScheme.contains("user@example.com"),
+        )
+    }
+
+    @Test
+    fun testHttpWithoutSpaces() {
+        // Multibyte characters around an email address should not produce URL/Link segments
+        val text =
+            "Vitor, vocêhttp://test.com? \uD83E\uDD7A"
+
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)
+
+        assertEquals(
+            "Vitor, você http://test.com? \uD83E\uDD7A",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
+        )
+    }
+
+    @Test
+    fun testHttpWithoutSpacesJapan() {
+        // Multibyte characters around an email address should not produce URL/Link segments
+        val text =
+            "Vitor, vocêhttp://test.comほげほげ"
+
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)
+
+        assertEquals(
+            "Vitor, você http://test.com ほげほげ",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
+        )
+    }
+
+    @Test
+    fun testHttpWithoutSpacesJapan2() {
+        // Multibyte characters around an email address should not produce URL/Link segments
+        val text =
+            "Vitor, vocêhttp://test.com。ほげほげ"
+
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)
+
+        assertEquals(
+            "Vitor, você http://test.com 。ほげほげ",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
        )
    }

@@ -65,11 +113,29 @@ class RichTextParserMultibyteTest {
        val text =
            "Vitor, você tem como colocar alguma forma de aviso se o link vai carregar uma imagem ou um vídeo? \uD83E\uDD7A"

-        val regex = Regex("(?<=[\\u0000-\\u00FF])(?=[\\u0100-\\uFFFF])|(?<=[\\u0100-\\uFFFF])(?=[\\u0000-\\u00FF])| +")
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)

        assertEquals(
-            "Vitor,-você-tem-como-colocar-alguma-forma-de-aviso-se-o-link-vai-carregar-uma-imagem-ou-um-vídeo?-\uD83E\uDD7A",
-            text.split(regex).joinToString("-"),
+            "Vitor, você tem como colocar alguma forma de aviso se o link vai carregar uma imagem ou um vídeo? \uD83E\uDD7A",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
+        )
+    }
+
+    @Test
+    fun testFullTextWithMultibyteAndQuotes() {
+        // Multibyte characters around an email address should not produce URL/Link segments
+        val text =
+            "I’ve been thinking lately about how I believe there will more than likely be models unattainable by most. Think Bloomberg Terminal. Where their cost of tokens is too high for the average lay person, but their level of “cognition” is unmatched by anything else. I’m sure there will even be many closed models that are invite only. Crazy times ahead."
+
+        val state =
+            RichTextParser()
+                .parseText(text, EmptyTagList, null)
+
+        assertEquals(
+            "I’ve been thinking lately about how I believe there will more than likely be models unattainable by most. Think Bloomberg Terminal. Where their cost of tokens is too high for the average lay person, but their level of “cognition” is unmatched by anything else. I’m sure there will even be many closed models that are invite only. Crazy times ahead.",
+            state.paragraphs.joinToString("\n") { it.words.joinToString(" ") { it.segmentText } },
        )
    }

@@ -91,9 +157,9 @@ class RichTextParserMultibyteTest {
        val state = RichTextParser().parseText(text, EmptyTagList, null)
        val allSegments = state.paragraphs.flatMap { it.words }

-        val urlSegments = allSegments.filterIsInstance<SchemelessUrlSegment>()
+        val urlSegments = allSegments.filterIsInstance<LinkSegment>()
        assertTrue("Should have SchemelessUrlSegment", urlSegments.isNotEmpty())
-        assertTrue("URL should be example.com", urlSegments.any { it.url == "example.com" })
+        assertTrue("URL should be example.com", urlSegments.any { it.segmentText == "https://example.com" })

        val textSegments = allSegments.filterIsInstance<RegularTextSegment>()
        assertTrue("Should have prefix ああ", textSegments.any { it.segmentText == "ああ" })
@@ -106,9 +172,9 @@ class RichTextParserMultibyteTest {
        val state = RichTextParser().parseText(text, EmptyTagList, null)
        val allSegments = state.paragraphs.flatMap { it.words }

-        val urlSegments = allSegments.filterIsInstance<SchemelessUrlSegment>()
+        val urlSegments = allSegments.filterIsInstance<LinkSegment>()
        assertTrue("Should have SchemelessUrlSegment", urlSegments.isNotEmpty())
-        assertTrue("URL should be example.com", urlSegments.any { it.url == "example.com" })
+        assertTrue("URL should be example.com", urlSegments.any { it.segmentText == "https://example.com" })

        val textSegments = allSegments.filterIsInstance<RegularTextSegment>()
        assertTrue("Should have suffix ああ", textSegments.any { it.segmentText == "ああ" })
@@ -0,0 +1,290 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.commons.richtext
+
+import kotlin.test.Ignore
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class UrlParserTest {
+    val parser = UrlParser()
+
+    fun test(
+        text: String,
+        expected: Urls,
+    ) {
+        val urlSet = parser.parseValidUrls(text)
+        assertEquals(expected.withScheme, urlSet.withScheme)
+        assertEquals(expected.withoutScheme, urlSet.withoutScheme)
+        assertEquals(expected.emails, urlSet.emails)
+    }
+
+    @Test
+    fun testSimpleText() =
+        test(
+            "test. com",
+            Urls(),
+        )
+
+    @Test
+    fun testBasicUrl() =
+        test(
+            "http://test.com",
+            Urls(withScheme = setOf("http://test.com")),
+        )
+
+    @Test
+    fun testNoSchemaUrl() =
+        test(
+            "test.com",
+            Urls(withoutScheme = setOf("test.com")),
+        )
+
+    @Test
+    fun testNoSchemaUrlPrefixMultibyte() =
+        test(
+            "ほtest.com",
+            Urls(withoutScheme = setOf("test.com")),
+        )
+
+    @Test
+    fun testNoSchemaUrlSuffixMultibyte() =
+        test(
+            "test.comほ",
+            Urls(withoutScheme = setOf("test.com")),
+        )
+
+    @Test
+    fun testNoSchemaUrlWithParams() =
+        test(
+            "test.com/some/me/hey?param=value#some=value",
+            Urls(withoutScheme = setOf("test.com/some/me/hey?param=value#some=value")),
+        )
+
+    @Test
+    fun testNoSchemaUrlWithParamsWithOtherWords() =
+        test(
+            "Hi there, check my website test.com/some/me/hey?param=value#some=value .",
+            Urls(withoutScheme = setOf("test.com/some/me/hey?param=value#some=value")),
+        )
+
+    @Test
+    fun testBasicUrlWithoutSpaceBefore() =
+        test(
+            "ahttp://test.com",
+            Urls(withScheme = setOf("http://test.com")),
+        )
+
+    @Test
+    fun testBasicUrlWithoutSpaceBeforeMultiByte() =
+        test(
+            "ほhttp://test.com",
+            Urls(withScheme = setOf("http://test.com")),
+        )
+
+    @Test
+    fun testBasicUrlWithoutSpaceAfter() =
+        test(
+            "http://test.comほ",
+            Urls(withScheme = setOf("http://test.com")),
+        )
+
+    @Test
+    fun testBasicUrlWithMultibytePath() =
+        test(
+            "http://test.com/ほ",
+            Urls(withScheme = setOf("http://test.com/ほ")),
+        )
+
+    @Test
+    fun testBasicUrls() =
+        test(
+            "http://test.com http://test2.com",
+            Urls(withScheme = setOf("http://test.com", "http://test2.com")),
+        )
+
+    @Test
+    fun testEmail() =
+        test(
+            "vitor@vitorpamplona.com",
+            Urls(emails = setOf("vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testEmailWithMultibytePrefix() =
+        test(
+            "ほvitor@vitorpamplona.com",
+            Urls(emails = setOf("vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testEmailWithMultibyteSuffix() =
+        test(
+            "vitor@vitorpamplona.comほ",
+            Urls(emails = setOf("vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testEmailWithMultibyteBoth() =
+        test(
+            "ほvitor@vitorpamplona.comほ",
+            Urls(emails = setOf("vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testUrlWithUserAndMultibyteSuffix() =
+        test(
+            "http://vitor@vitorpamplona.comほ",
+            Urls(withScheme = setOf("http://vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testUrlWithUserAndMultibytePrefix() =
+        test(
+            "ほhttp://vitor@vitorpamplona.com",
+            Urls(withScheme = setOf("http://vitor@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testNostrUrls() =
+        test(
+            "nostr:npub1aabbcc",
+            Urls(),
+        )
+
+    @Test
+    fun testUrlsWithUsernameAndPath() =
+        test(
+            "miceliomad@miceliomad.github.io/nostr/",
+            Urls(withoutScheme = setOf("miceliomad@miceliomad.github.io/nostr/")),
+        )
+
+    @Test
+    fun testUrlsWithQuery() =
+        test(
+            " universe.nostrich.land?lang=zh ",
+            Urls(withoutScheme = setOf("universe.nostrich.land?lang=zh")),
+        )
+
+    @Test
+    fun testUrlsWithSchemaPathAndQuery() =
+        test(
+            "https://miceliomad.github.io/nostr/test?me=you",
+            Urls(withScheme = setOf("https://miceliomad.github.io/nostr/test?me=you")),
+        )
+
+    @Test
+    fun testUrlsWithPathAndQuery() =
+        test(
+            "miceliomad.github.io/nostr/test?me=you",
+            Urls(withoutScheme = setOf("miceliomad.github.io/nostr/test?me=you")),
+        )
+
+    @Test
+    fun testAvifFileNameComplete() =
+        test(
+            "https://bae.st/media/66b08dde784287ed8f92c455bc62076a04671ccb44097550626a532185a5d3ed.avif?name=81ca16-b665-4f57-80cb-11a58461fb61.avif",
+            Urls(withScheme = setOf("https://bae.st/media/66b08dde784287ed8f92c455bc62076a04671ccb44097550626a532185a5d3ed.avif?name=81ca16-b665-4f57-80cb-11a58461fb61.avif")),
+        )
+
+    @Test
+    fun testAvifFileName() =
+        test(
+            "81ca16-b665-4f57-80cb-11a58461fb61.avif",
+            Urls(withoutScheme = setOf("81ca16-b665-4f57-80cb-11a58461fb61.avif")),
+        )
+
+    @Test
+    fun testMultiLine() =
+        test(
+            """
+            22.8K (3.2%) nos.lol
+            22.7K (3.1%) universe.nostrich.land?lang=zh
+            22.5K (3.1%) universe.nostrich.land?lang=en
+            """.trimIndent(),
+            Urls(withoutScheme = setOf("nos.lol", "universe.nostrich.land?lang=zh", "universe.nostrich.land?lang=en")),
+        )
+
+    @Test
+    fun testEmailWithDashes() =
+        test(
+            "freeverification@Nostr-Check.com",
+            Urls(emails = setOf("freeverification@Nostr-Check.com")),
+        )
+
+    @Test
+    fun testEmailWithManyDashes() =
+        test(
+            "free-veri-fica-tion@No-str-Ch-eck.com",
+            Urls(emails = setOf("free-veri-fica-tion@No-str-Ch-eck.com")),
+        )
+
+    @Test
+    fun testEmailWithUnderscore() =
+        test(
+            "vi_t_or@vitorpamplona.com",
+            Urls(emails = setOf("vi_t_or@vitorpamplona.com")),
+        )
+
+    @Test
+    fun testEmailsWithPeriod() =
+        test(
+            "john.smith@gmail.com",
+            Urls(emails = setOf("john.smith@gmail.com")),
+        )
+
+    @Test
+    fun testStrangeError() =
+        test(
+            "neomobius_at_mstdn.jp@mostr.pub",
+            Urls(emails = setOf("neomobius_at_mstdn.jp@mostr.pub")),
+        )
+
+    @Test
+    @Ignore("We need to make this work")
+    fun testRelayUrl() =
+        test(
+            "wss://test.com",
+            Urls(withScheme = setOf("wss://test.com")),
+        )
+
+    @Test
+    @Ignore("We need to make this work")
+    fun testBech12() =
+        test(
+            "nostr:npub1aabbcc",
+            Urls(withScheme = setOf("wss://test.com")),
+        )
+
+    @Test
+    fun testJapaneseUrls() =
+        test(
+            "我进入你的主页很卡顿，也许是你的关注人数或者其他数据太多了，其他人主页没有这么卡顿。来自amethyst客户端",
+            Urls(withScheme = emptySet()),
+        )
+
+    @Test
+    fun testHour() =
+        test(
+            "10.00hr,",
+            Urls(withScheme = emptySet()),
+        )
+}