Switches to our own version of the Url Detector

commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt

@@ -20,8 +20,6 @@
  */
 package com.vitorpamplona.amethyst.commons.richtext
 
-import com.linkedin.urls.detection.UrlDetector
-import com.linkedin.urls.detection.UrlDetectorOptions
 import com.vitorpamplona.amethyst.commons.emojicoder.EmojiCoder
 import com.vitorpamplona.amethyst.commons.model.ImmutableListOfLists
 import com.vitorpamplona.quartz.experimental.inlineMetadata.Nip54InlineMetadata
@@ -39,7 +37,6 @@ import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.toImmutableList
 import kotlinx.collections.immutable.toImmutableMap
-import kotlinx.collections.immutable.toImmutableSet
 import kotlinx.collections.immutable.toPersistentList
 import java.net.MalformedURLException
 import java.net.URISyntaxException
@@ -101,27 +98,43 @@ class RichTextParser {
         }
     }
 
-    fun parseValidUrls(content: String): LinkedHashSet<String> {
-        val urls = UrlDetector(content, UrlDetectorOptions.Default).detect()
+    fun fixMissingSpaces(
+        input: String,
+        urlList: Set<String>,
+    ): String {
+        if (urlList.isEmpty()) return input
 
-        return urls.mapNotNullTo(LinkedHashSet(urls.size)) {
-            if (it.originalUrl.contains("@")) {
-                if (Patterns.EMAIL_ADDRESS.matches(it.originalUrl)) {
-                    null
-                } else {
-                    it.originalUrl
-                }
-            } else if (isNumber(it.originalUrl)) {
-                null // avoids urls that look like 123.22
-            } else if (it.originalUrl.contains("。")) {
-                null // avoids Japanese characters as fake urls
-            } else {
-                if (HTTPRegex.matches(it.originalUrl)) {
-                    it.originalUrl
-                } else {
-                    null
-                }
+        // Escape and join words: (word1|word2)
+        val wordsPattern = urlList.sortedByDescending { it.length }.joinToString("|") { Regex.escape(it) }
+
+        // Regex breakdown:
+        // ([^ ])?          -> Group 1: Optional character that is NOT a space or new line (Prefix)
+        // ($wordsPattern)  -> Group 2: One of your target words
+        // ([^ ])?          -> Group 3: Optional character that is NOT a space or new line (Suffix)
+        val regex = Regex("([^ \n])?($wordsPattern)([^ \n])?")
+
+        return regex.replace(input) { match ->
+            val prefix = match.groups[1]?.value ?: ""
+            val word = match.groups[2]?.value ?: ""
+            val suffix = match.groups[3]?.value ?: ""
+
+            val result = StringBuilder()
+
+            // Add prefix + space if the prefix exists
+            if (prefix.isNotEmpty()) {
+                result.append(prefix)
+                result.append(" ")
             }
+
+            result.append(word)
+
+            // Add space + suffix if the suffix exists
+            if (suffix.isNotEmpty()) {
+                result.append(" ")
+                result.append(suffix)
+            }
+
+            result.toString()
         }
     }
 
@@ -131,30 +144,41 @@ class RichTextParser {
         callbackUri: String?,
     ): RichTextViewerState {
         val imetas = tags.lists.imetasByUrl()
-        val urlSet = parseValidUrls(content)
+        val urlSet = UrlParser().parseValidUrls(content)
 
-        val imagesForPager =
-            urlSet.mapNotNull { fullUrl -> createMediaContent(fullUrl, imetas, content, callbackUri) }.associateBy { it.url }
+        val mediaContents =
+            urlSet.withScheme.mapNotNull { fullUrl ->
+                createMediaContent(fullUrl, imetas, content, callbackUri)
+            } +
+                urlSet.withoutScheme.mapNotNull { fullUrl ->
+                    createMediaContent(fullUrl, imetas, content, callbackUri)
+                }
 
-        val imageUrls = imagesForPager.filterValues { it is MediaUrlImage }.keys
-        val videoUrls = imagesForPager.filterValues { it is MediaUrlVideo }.keys
+        val mediaForPager = mediaContents.associateBy { it.url }
+
+        val imageUrls = mediaForPager.filterValues { it is MediaUrlImage }.keys
+        val videoUrls = mediaForPager.filterValues { it is MediaUrlVideo }.keys
 
         val emojiMap = CustomEmoji.createEmojiMap(tags.lists)
 
-        val segments = findTextSegments(content, imageUrls, videoUrls, urlSet, emojiMap, tags)
+        val allUrls = urlSet.withScheme + urlSet.withoutScheme + urlSet.emails
 
-        val base64Images = segments.map { it.words.filterIsInstance<Base64Segment>() }.flatten()
+        val newContent = fixMissingSpaces(content, allUrls)
 
-        val imagesForPagerWithBase64 =
-            imagesForPager +
+        val segments = findTextSegments(newContent, imageUrls, videoUrls, urlSet, emojiMap, tags)
+
+        val base64Images = segments.flatMap { it.words.filterIsInstance<Base64Segment>() }
+
+        val mediaForPagerWithBase64 =
+            mediaForPager +
                 base64Images
                     .mapNotNull { createMediaContent(it.segmentText, emptyMap(), content, callbackUri) }
                     .associateBy { it.url }
 
         return RichTextViewerState(
-            urlSet.toImmutableSet(),
-            imagesForPagerWithBase64.toImmutableMap(),
-            imagesForPagerWithBase64.values.toImmutableList(),
+            urlSet,
+            mediaForPagerWithBase64.toImmutableMap(),
+            mediaForPagerWithBase64.values.toImmutableList(),
             emojiMap.toImmutableMap(),
             segments,
             tags,
@@ -165,7 +189,7 @@ class RichTextParser {
         content: String,
         images: Set<String>,
         videos: Set<String>,
-        urls: Set<String>,
+        urls: Urls,
         emojis: Map<String, String>,
         tags: ImmutableListOfLists<String>,
     ): ImmutableList<ParagraphState> {
@@ -175,18 +199,14 @@ class RichTextParser {
         lines.forEach { paragraph ->
             val isRTL = isArabic(paragraph)
 
-            val wordList = paragraph.trimEnd().split(wordBoundaryRegex).filter { it.isNotEmpty() }
+            val wordList = paragraph.trimEnd().split(' ')
 
-            if (wordList.isEmpty()) {
-                paragraphSegments.add(ParagraphState(persistentListOf(RegularTextSegment("")), isRTL))
-            } else {
-                val segments = ArrayList<Segment>(wordList.size)
-                wordList.forEach { word ->
-                    segments.add(wordIdentifier(word, images, videos, urls, emojis, tags))
-                }
-
-                paragraphSegments.add(ParagraphState(segments.toPersistentList(), isRTL))
+            val segments = ArrayList<Segment>(wordList.size)
+            wordList.forEach { word ->
+                segments.add(wordIdentifier(word, images, videos, urls, emojis, tags))
             }
+
+            paragraphSegments.add(ParagraphState(segments.toPersistentList(), isRTL))
         }
 
         val segmentsWithGalleries = GalleryParser().processParagraphs(paragraphSegments)
@@ -204,8 +224,6 @@ class RichTextParser {
             }.toImmutableList()
     }
 
-    private fun isNumber(word: String) = numberPattern.matches(word)
-
     private fun isPhoneNumberChar(c: Char): Boolean =
         when (c) {
             in '0'..'9' -> true
@@ -236,7 +254,7 @@ class RichTextParser {
         word: String,
         images: Set<String>,
         videos: Set<String>,
-        urls: Set<String>,
+        urls: Urls,
         emojis: Map<String, String>,
         tags: ImmutableListOfLists<String>,
     ): Segment {
@@ -246,13 +264,29 @@ class RichTextParser {
             if (Patterns.BASE64_IMAGE.matches(word)) return Base64Segment(word)
         }
 
-        if (images.contains(word)) return ImageSegment(word)
+        if (images.contains(word)) {
+            return if (urls.withoutScheme.contains(word)) {
+                ImageSegment("https://$word")
+            } else {
+                ImageSegment(word)
+            }
+        }
 
-        if (videos.contains(word)) return VideoSegment(word)
+        if (videos.contains(word)) {
+            return if (urls.withoutScheme.contains(word)) {
+                VideoSegment("https://$word")
+            } else {
+                VideoSegment(word)
+            }
+        }
 
         if (word.startsWith("ws://") || word.startsWith("wss://")) return RelayUrlSegment(word)
 
-        if (urls.contains(word)) return LinkSegment(word)
+        if (urls.withoutScheme.contains(word)) {
+            return LinkSegment("https://$word")
+        } else if (urls.withScheme.contains(word)) {
+            return LinkSegment(word)
+        }
 
         if (CustomEmoji.fastMightContainEmoji(word, emojis) && emojis.any { word.contains(it.key) }) return EmojiSegment(word)
 
@@ -262,13 +296,11 @@ class RichTextParser {
 
         if (word.startsWith("cashuA", true) || word.startsWith("cashuB", true)) return CashuSegment(word)
 
-        if (word.startsWith("#")) return parseHash(word, tags)
+        if (word.startsWith('#')) return parseHash(word, tags)
 
         if (EmojiCoder.isCoded(word)) return SecretEmoji(word)
 
-        if (word.contains("@")) {
-            if (Patterns.EMAIL_ADDRESS.matches(word)) return EmailSegment(word)
-        }
+        if (urls.emails.contains(word)) return EmailSegment(word)
 
         if (startsWithNIP19Scheme(word)) return BechSegment(word)
 
@@ -276,18 +308,6 @@ class RichTextParser {
             if (Patterns.PHONE.matches(word)) return PhoneSegment(word)
         }
 
-        val indexOfPeriod = word.indexOf(".")
-        if (indexOfPeriod > 0 && indexOfPeriod < word.length - 1) { // periods cannot be the last one
-            val schemelessMatcher = noProtocolUrlValidator.find(word)
-            if (schemelessMatcher != null) {
-                val url = schemelessMatcher.groups[1]?.value // url
-                val additionalChars = schemelessMatcher.groups[4]?.value?.ifEmpty { null } // additional chars
-                if (additionalUrlSchema.find(word) != null && url != null) {
-                    return SchemelessUrlSegment(word, url, additionalChars)
-                }
-            }
-        }
-
         return RegularTextSegment(word)
     }
 
@@ -343,7 +363,7 @@ class RichTextParser {
 
         val noProtocolUrlValidator =
             Regex(
-                "(([a-zA-Z0-9_-]+\\.)*[a-zA-Z][a-zA-Z0-9_-]+[\\.\\:][a-zA-Z0-9_]+([\\/ \\?\\=\\&\\#\\.]?[a-zA-Z0-9_-]+)*\\/?)(.*)",
+                "(([a-zA-Z0-9_-]+@)?([a-zA-Z0-9_-]+\\.)*[a-zA-Z0-9_-]+[\\.\\:][a-zA-Z0-9_]+([\\/ \\?\\=\\&\\#\\.]?[a-zA-Z0-9_-]+)*\\/?)(.*)",
             )
 
         // Splits at spaces AND at ASCII/multibyte character boundaries

commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserSegments.kt

@@ -24,11 +24,10 @@ import androidx.compose.runtime.Immutable
 import com.vitorpamplona.amethyst.commons.model.ImmutableListOfLists
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.ImmutableMap
-import kotlinx.collections.immutable.ImmutableSet
 
 @Immutable
 class RichTextViewerState(
-    val urlSet: ImmutableSet<String>,
+    val urlSet: Urls,
     val imagesForPager: ImmutableMap<String, MediaUrlContent>,
     val imageList: ImmutableList<MediaUrlContent>,
     val customEmoji: ImmutableMap<String, String>,
@@ -139,13 +138,6 @@ class HashTagSegment(
     val extras: String?,
 ) : Segment(segment)
 
-@Immutable
-class SchemelessUrlSegment(
-    segment: String,
-    val url: String,
-    val extras: String?,
-) : Segment(segment)
-
 @Immutable
 class RelayUrlSegment(
     segment: String,

commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParser.kt

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.commons.richtext
+
+import com.vitorpamplona.amethyst.commons.richtext.RichTextParser.Companion.noProtocolUrlValidator
+import com.vitorpamplona.quartz.utils.urldetector.Url
+import com.vitorpamplona.quartz.utils.urldetector.detection.UrlDetector
+
+class Urls(
+    val withScheme: Set<String> = emptySet(),
+    val withoutScheme: Set<String> = emptySet(),
+    val emails: Set<String> = emptySet(),
+)
+
+class UrlParser {
+    fun Char.isAsciiLetter(): Boolean = (this in 'a'..'z' || this in 'A'..'Z')
+
+    fun Url.isValidTopLevelDomain(): Boolean {
+        /*
+        According to the TLD Applicant Guidebook published June 2012, ICANN does not allow numbers in TLDs.
+         */
+        val startOfTopDomain = host.lastIndexOf('.') + 1
+        return if (startOfTopDomain < host.length) {
+            val topLevelDomain = host.substring(startOfTopDomain)
+            topLevelDomain.isNotEmpty() && topLevelDomain[0].isAsciiLetter()
+        } else {
+            false
+        }
+    }
+
+    fun Url.wroteWithSchema(): Boolean = originalUrl.startsWith(scheme)
+
+    fun Url.isEmail(): Boolean = originalUrl.contains('@') && path == "/" && query.isEmpty() && fragment.isEmpty()
+
+    fun Char.isValidLastHostnameChar(): Boolean = (this in 'a'..'z' || this in 'A'..'Z' || this in '0'..'9')
+
+    fun Url.isValidLastHostnameChar(): Boolean = host[host.length - 1].isValidLastHostnameChar()
+
+    fun Url.endsWithHost(): Boolean = originalUrl.endsWith(host)
+
+    val notAHostNameChar = "[^a-zA-Z0-9.-]".toRegex()
+
+    fun parseValidUrls(content: String): Urls {
+        val urls = UrlDetector(content).detect()
+
+        val completeUrls = mutableSetOf<String>()
+        val urlsWithoutScheme = mutableSetOf<String>()
+        val emails = mutableSetOf<String>()
+
+        println("AABBBCC parseValidUrls ${urls.size}")
+
+        urls.forEach {
+            println("AABBBCC Testing ${it.originalUrl}")
+            if (it.isValidTopLevelDomain()) {
+                if (it.wroteWithSchema()) {
+                    if (it.isValidLastHostnameChar()) {
+                        completeUrls.add(it.originalUrl)
+                    } else if (it.endsWithHost()) {
+                        val match = notAHostNameChar.find(it.host)
+                        if (match != null) {
+                            completeUrls.add(it.originalUrl.substring(0, (it.originalUrl.length - it.host.length) + match.range.first))
+                        } else {
+                            completeUrls.add(it.originalUrl)
+                        }
+                    }
+                } else {
+                    // emails are understood as urls from the detector.
+                    if (it.isEmail()) {
+                        Patterns.EMAIL_ADDRESS.findAll(it.originalUrl).forEach {
+                            emails.add(it.value)
+                        }
+                    } else {
+                        noProtocolUrlValidator.findAll(it.originalUrl).forEach { components ->
+                            val url = components.groups[1]?.value
+                            if (url != null) {
+                                urlsWithoutScheme.add(url)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return Urls(
+            withScheme = completeUrls,
+            withoutScheme = urlsWithoutScheme,
+            emails = emails,
+        )
+    }
+}