Adds single label domains to parse nostr: urls too

commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParser.kt

@@ -65,10 +65,7 @@ class UrlParser {
         val urlsWithoutScheme = mutableSetOf<String>()
         val emails = mutableSetOf<String>()
 
-        println("AABBBCC parseValidUrls ${urls.size}")
-
         urls.forEach {
-            println("AABBBCC Testing ${it.originalUrl}")
             if (it.isValidTopLevelDomain()) {
                 if (it.wroteWithSchema()) {
                     if (it.isValidLastHostnameChar()) {

commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParserTest.kt

@@ -20,7 +20,6 @@
  */
 package com.vitorpamplona.amethyst.commons.richtext
 
-import kotlin.test.Ignore
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -167,7 +166,7 @@ class UrlParserTest {
     fun testNostrUrls() =
         test(
             "nostr:npub1aabbcc",
-            Urls(),
+            Urls(withScheme = setOf("nostr:npub1aabbcc")),
         )
 
     @Test
@@ -259,7 +258,6 @@ class UrlParserTest {
         )
 
     @Test
-    @Ignore("We need to make this work")
     fun testRelayUrl() =
         test(
             "wss://test.com",
@@ -267,11 +265,10 @@ class UrlParserTest {
         )
 
     @Test
-    @Ignore("We need to make this work")
     fun testBech12() =
         test(
             "nostr:npub1aabbcc",
-            Urls(withScheme = setOf("wss://test.com")),
+            Urls(withScheme = setOf("nostr:npub1aabbcc")),
         )
 
     @Test

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarker.kt

@@ -47,6 +47,12 @@ class UrlMarker {
         }
     }
 
+    fun hasScheme() = schemeIndex >= 0
+
+    fun hasPort() = portIndex >= 0
+
+    fun hasUsernamePassword() = usernamePasswordIndex >= 0
+
     /**
      * @param urlPart The part you want the index of
      * @return Returns the index of the part

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/DomainNameReader.kt

@@ -87,14 +87,6 @@ class DomainNameReader(
         ReadUserPass,
     }
 
-    /**
-     * The interface that gets called for each character that's non-matching (to a valid domain name character) in to count
-     * the matching quotes and parenthesis correctly.
-     */
-    interface CharacterHandler {
-        fun addCharacter(character: Char)
-    }
-
     /**
      * Keeps track the number of dots that were found in the domain name.
      */
@@ -136,6 +128,12 @@ class DomainNameReader(
      */
     private var zoneIndex = false
 
+    var isIpV4 = false
+        private set
+
+    var isIpV6 = false
+        private set
+
     fun String.isDotPercent() = this == "%2e" || this == "%2E"
 
     /**
@@ -207,14 +205,6 @@ class DomainNameReader(
                     }
                 } else if (isAlpha(curr) || curr == '-' || curr.code >= INTERNATIONAL_CHAR_START) {
                     numeric = false
-                } else if (!isNumeric(curr)) {
-                    // if its not _numeric and not alphabetical, then restart searching for a domain from this point.
-                    newStart = index + 1
-                    currentLabelLength = 0
-                    topLevelLength = 0
-                    numeric = true
-                    dots = 0
-                    done = true
                 }
                 index++
             }
@@ -408,6 +398,8 @@ class DomainNameReader(
             this[2] == '-' &&
             this[3] == '-'
 
+    fun labelCount() = dots + (if (currentLabelLength > 0) 1 else 0)
+
     /**
      * Checks the current state of this object and returns if the valid state indicates that the
      * object has a valid domain name. If it does, it will return append the last character
@@ -440,15 +432,23 @@ class DomainNameReader(
         val domainLength: Int =
             buffer.length - startDomainName + (if (currentLabelLength > 0) lastDotLength else 0)
         val dotCount = dots + (if (currentLabelLength > 0) 1 else 0)
-        if (domainLength >= MAX_DOMAIN_LENGTH || (dotCount > MAX_NUMBER_LABELS)) {
+        if (domainLength >= MAX_DOMAIN_LENGTH || dotCount > MAX_NUMBER_LABELS) {
             valid = false
         } else if (numeric) {
             val testDomain = buffer.substring(startDomainName).lowercase()
             valid = isValidIpv4(testDomain)
+            if (valid) {
+                isIpV4 = true
+            }
         } else if (seenBracket) {
             val testDomain = buffer.substring(startDomainName).lowercase()
             valid = isValidIpv6(testDomain)
-        } else if ((currentLabelLength > 0 && dots >= 1) || (dots >= 2 && currentLabelLength == 0)) {
+            if (valid) {
+                isIpV6 = true
+            }
+        } else if (buffer.isNotEmpty() && buffer.last() == ':') {
+            valid = false
+        } else if ((currentLabelLength > 0 && dots >= 1) || (dots >= 2 && currentLabelLength == 0) || (dots == 0)) {
             var topStart: Int = buffer.length - topLevelLength
             if (currentLabelLength == 0) {
                 topStart--
@@ -472,10 +472,6 @@ class DomainNameReader(
             return validState
         }
 
-        // Roll back one char if its invalid to handle: "00:41.<br />"
-        // This gets detected as 41.br otherwise.
-        reader.goBack()
-
         // return invalid state.
         return ReaderNextState.InvalidDomainName
     }

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt

@@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.utils.urldetector.Url
 import com.vitorpamplona.quartz.utils.urldetector.UrlMarker
 import com.vitorpamplona.quartz.utils.urldetector.UrlPart
 import kotlin.math.max
+import kotlin.text.deleteRange
 
 class UrlDetector(
     content: String,
@@ -44,9 +45,9 @@ class UrlDetector(
     private var hasScheme = false
 
     /**
-     * If the first character in the url is a quote, then look for matching quote at the end.
+     * has Multi-level labels
      */
-    private var quoteStart = false
+    private var isSingleLevelLabel = false
 
     /**
      * Stores the found urls.
@@ -62,14 +63,7 @@ class UrlDetector(
      * The states to use to continue writing or not.
      */
     enum class ReadEndState {
-        /**
-         * The current url is valid.
-         */
         ValidUrl,
-
-        /**
-         * The current url is invalid.
-         */
         InvalidUrl,
     }
 
@@ -95,6 +89,15 @@ class UrlDetector(
             // read the next char to process.
             when (val curr = reader.read()) {
                 ' ' -> {
+                    // space was found, check if it's a valid single level domain.
+                    if (buffer.isNotEmpty() && hasScheme) {
+                        reader.goBack()
+                        val domain = buffer.substring(length)
+                        if (!readDomainName(domain)) {
+                            readEnd(ReadEndState.InvalidUrl)
+                        }
+                    }
+
                     buffer.append(curr)
                     readEnd(ReadEndState.InvalidUrl)
                     length = 0
@@ -160,7 +163,7 @@ class UrlDetector(
                 '/' -> {
                     // "/" was found, then we either read a scheme, or if we already read a scheme, then
                     // we are reading a url in the format http://123123123/asdf
-                    if (hasScheme) {
+                    if (hasScheme || buffer.length > 1) {
                         // we already have the scheme, so then we already read:
                         // http://something/ <- if something is all numeric then its a valid url.
                         // OR we are searching for single level domains. We have buffer length > 1 condition
@@ -202,6 +205,13 @@ class UrlDetector(
 
             position = reader.position
         }
+
+        // check if it's a valid single level domain.
+        if (buffer.isNotEmpty() && hasScheme) {
+            if (!readDomainName(buffer.substring(length))) {
+                readEnd(ReadEndState.InvalidUrl)
+            }
+        }
     }
 
     /**
@@ -218,7 +228,7 @@ class UrlDetector(
                 reader.goBack()
 
                 // Check buffer length before clearing it; set length to 0 if buffer is empty
-                if (buffer.length > 0) {
+                if (buffer.isNotEmpty()) {
                     buffer.deleteRange(buffer.length - 1, buffer.length)
                 } else {
                     length = 0
@@ -237,6 +247,13 @@ class UrlDetector(
         } else if (readScheme() && buffer.isNotEmpty()) {
             hasScheme = true
             length = buffer.length // set length to be right after the scheme
+        } else if (buffer.isNotEmpty() && reader.canReadChars(1)) {
+            // takes care of case like hi:
+            reader.goBack() // unread the ":" so readDomainName can take care of the port
+            buffer.deleteAt(buffer.length - 1)
+            if (!readDomainName(buffer.toString())) {
+                readEnd(ReadEndState.InvalidUrl)
+            }
         } else {
             readEnd(ReadEndState.InvalidUrl)
             length = 0
@@ -302,6 +319,21 @@ class UrlDetector(
             } else if (curr == '[') { // if we're starting to see an ipv6 address
                 reader.goBack() // unread the '[', so that we can start looking for ipv6
                 return false
+            } else if (originalLength > 0 && numSlashes == 0 && CharUtils.isAlpha(curr)) {
+                // If we had already read something before the : and we are matching regardless of slashes, assume it's a scheme
+
+                // Add the slashes to the end of the scheme so it matches what's in the scheme list
+                val schemeStartIndex = findValidSchemeNoSlashesStartIndex(buffer.toString())
+                if (schemeStartIndex >= 0) {
+                    buffer.deleteRange(0, schemeStartIndex)
+                    currentUrlMarker.setIndex(UrlPart.SCHEME, 0)
+                    reader.goBack()
+                    return true
+                } else {
+                    reader.goBack()
+                    return readUserPass(0)
+                }
+                // If this didn't match a defined scheme, continue processing as usual
             } else if (originalLength > 0 || numSlashes > 0 || !CharUtils.isAlpha(curr)) {
                 // if it's not a character a-z or A-Z then assume we aren't matching scheme, but instead
                 // matching username and password.
@@ -321,6 +353,14 @@ class UrlDetector(
             .firstOrNull() ?: -1
     }
 
+    private fun findValidSchemeNoSlashesStartIndex(optionalScheme: String): Int {
+        val optionalSchemeLowercase = optionalScheme.lowercase()
+        return VALID_SCHEMES_NO_SLASHES
+            .filter(optionalSchemeLowercase::endsWith)
+            .map(optionalSchemeLowercase::lastIndexOf)
+            .firstOrNull() ?: -1
+    }
+
     /**
      * Reads the input and looks for a username and password.
      * Handles:
@@ -390,6 +430,9 @@ class UrlDetector(
 
         // Try to read the dns and act on the response.
         val state = reader.readDomainName()
+
+        isSingleLevelLabel = reader.labelCount() <= 1 && !reader.isIpV4 && !reader.isIpV6
+
         return when (state) {
             DomainNameReader.ReaderNextState.ValidDomainName -> {
                 readEnd(ReadEndState.ValidUrl)
@@ -481,9 +524,27 @@ class UrlDetector(
         while (!reader.eof()) {
             // read the next one and remember the length
             val curr = reader.read()
+
+            // requires at least one number as port to
+            // better handle http://http://
+            if (portLen == 0 && isSingleLevelLabel) {
+                if (!CharUtils.isNumeric(curr)) {
+                    reader.goBack()
+
+                    currentUrlMarker.unsetIndex(UrlPart.PORT)
+                    return readEnd(ReadEndState.InvalidUrl)
+                }
+            }
+
             portLen++
 
-            if (curr == '/') {
+            if (curr == ':') {
+                // rejects a second port
+                reader.goBack()
+
+                currentUrlMarker.unsetIndex(UrlPart.PORT)
+                return readEnd(ReadEndState.InvalidUrl)
+            } else if (curr == '/') {
                 // continue to read path
                 buffer.append(curr)
                 return readPath()
@@ -522,13 +583,27 @@ class UrlDetector(
      */
     private fun readPath(): Boolean {
         currentUrlMarker.setIndex(UrlPart.PATH, buffer.length - 1)
+
+        var endsOnASlash = true
+
         while (!reader.eof()) {
             // read the next char
             val curr = reader.read()
 
             if (curr == ' ') {
-                // if end of state and we got here, then the url is valid.
+                // if end of state and we got here, then the url is valid
-                return readEnd(ReadEndState.ValidUrl)
+                // if it is not just a word/word
+                if (
+                    currentUrlMarker.hasScheme() ||
+                    currentUrlMarker.hasPort() ||
+                    currentUrlMarker.hasUsernamePassword() ||
+                    !isSingleLevelLabel ||
+                    endsOnASlash
+                ) {
+                    return readEnd(ReadEndState.ValidUrl)
+                } else {
+                    return readEnd(ReadEndState.InvalidUrl)
+                }
             }
 
             // append the char
@@ -542,10 +617,26 @@ class UrlDetector(
                 // if # read the fragment
                 return readFragment()
             }
+
+            endsOnASlash = curr == '/'
         }
 
         // end of input then this url is good.
-        return readEnd(ReadEndState.ValidUrl)
+        // if end of state and we got here, then the url is valid
+        // if it is not just a word/word
+        // no need to check for query and fragments
+        // here we accept urls that end in /
+        if (
+            currentUrlMarker.hasScheme() ||
+            currentUrlMarker.hasPort() ||
+            currentUrlMarker.hasUsernamePassword() ||
+            !isSingleLevelLabel ||
+            endsOnASlash
+        ) {
+            return readEnd(ReadEndState.ValidUrl)
+        } else {
+            return readEnd(ReadEndState.InvalidUrl)
+        }
     }
 
     /**
@@ -556,12 +647,6 @@ class UrlDetector(
     private fun readEnd(state: ReadEndState?): Boolean {
         // if the url is valid and greater then 0
         if (state == ReadEndState.ValidUrl && buffer.isNotEmpty()) {
-            // get the last character. if its a quote, cut it off.
-            val len: Int = buffer.length
-            if (quoteStart && buffer[len - 1] == '\"') {
-                buffer.deleteRange(len - 1, len)
-            }
-
             // Add the url to the list of good urls.
             if (buffer.isNotEmpty()) {
                 currentUrlMarker.originalUrl = buffer.toString()
@@ -573,7 +658,6 @@ class UrlDetector(
         buffer.deleteRange(0, buffer.length)
 
         // reset the state of internal objects.
-        quoteStart = false
         hasScheme = false
         currentUrlMarker = UrlMarker()
 
@@ -582,16 +666,21 @@ class UrlDetector(
     }
 
     companion object {
-        private val VALID_SCHEMES: List<String> =
+        private val VALID_SCHEMES_NO_SLASHES: List<String> =
             listOf(
-                "http://",
+                "http:",
-                "https://",
+                "https:",
-                "ftp://",
+                "ftp:",
-                "ftps://",
+                "ftps:",
-                "ws://",
+                "ws:",
-                "wss://",
+                "wss:",
-                // "nostr:",
+                "nostr:",
-                // "blossom:",
+                "blossom:",
             )
+
+        private val VALID_SCHEMES =
+            VALID_SCHEMES_NO_SLASHES.map {
+                "$it//"
+            }
     }
 }

quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt

@@ -21,6 +21,7 @@
 package com.vitorpamplona.quartz.utils.urldetector.detection
 
 import com.vitorpamplona.quartz.utils.urldetector.Url
+import kotlinx.coroutines.test.runTest
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -134,6 +135,7 @@ class UriDetectionTest {
             "Do newlines and tabs break? google.com/hello/\nworld www.yahoo.com\t/stuff/ yahoo.com/\thello news.ycombinator.com\u0000/hello world",
             "google.com/hello/",
             "www.yahoo.com",
+            "stuff/",
             "yahoo.com/",
             "news.ycombinator.com",
         )
@@ -335,6 +337,10 @@ class UriDetectionTest {
         )
         runTest(
             "browsers dont support this without a scheme look 0xC00002EB/bobo",
+            "0xC00002EB/bobo",
+        )
+        runTest(
+            "browsers dont support this without a scheme look test/bobo",
         )
     }
 
@@ -350,11 +356,19 @@ class UriDetectionTest {
         )
         runTest(
             "browsers dont support this without a scheme look 030000001353/bobo",
+            "030000001353/bobo",
+        )
+        runTest(
+            "browsers dont support this without a scheme look 1727123/bobo",
         )
     }
 
     @Test
     fun testUrlWithEmptyPort() {
+        runTest(
+            "http://wtfismyip.com:/foo.html",
+            "http://wtfismyip.com:/foo.html",
+        )
         runTest(
             "http://wtfismyip.com://foo.html",
             "http://wtfismyip.com://foo.html",
@@ -465,7 +479,9 @@ class UriDetectionTest {
         runTest("[b[::7f8e]:55]akjef[::]", "[::7f8e]:55", "[::]")
         runTest(
             "[bcad::kkkk:aaaa:3dd0[::7f8e]:57b7:34d5]akjef[::]",
+            "aaaa:3",
             "[::7f8e]:57",
+            "b7:34",
             "[::]",
         )
     }
@@ -550,6 +566,7 @@ class UriDetectionTest {
             "Do newlines and tabs break? [::2e80:0:0]/hello/\nworld [::BEEF:ADD:BEEF]\t/stuff/ [AAbb:AAbb:AAbb::]/\thello [::2e80:0:0\u0000]/hello world",
             "[::2e80:0:0]/hello/",
             "[::BEEF:ADD:BEEF]",
+            "stuff/",
             "[AAbb:AAbb:AAbb::]/",
         )
     }
@@ -622,17 +639,102 @@ class UriDetectionTest {
         val parser = UrlDetector("ftp:example.com")
         val found: List<Url> = parser.detect()
         for (url in found) {
-            assertEquals(url.scheme, "https")
+            assertEquals(url.scheme, "ftp")
             // Should be detected as a username now and set to default http://
             assertEquals(url.host, "example.com")
         }
     }
 
+    @Test
+    fun testSingleLevelDomain() {
+        runTest("http://localhost:9000/lalala hehe", "http://localhost:9000/lalala")
+        runTest("localhost:9000/lalala hehe", "localhost:9000/lalala")
+        runTest("http://localhost lasdf", "http://localhost")
+        runTest("localhost:9000/lalala", "localhost:9000/lalala")
+        runTest("192.168.1.1/lalala", "192.168.1.1/lalala")
+        runTest("http://localhost", "http://localhost")
+        runTest("//localhost", "//localhost")
+        runTest("asf//localhost")
+        runTest("hello/", "hello/")
+        runTest("hello/ ", "hello/")
+        runTest("hello")
+        runTest("go/", "go/")
+        runTest("hello:password@go12//", "hello:password@go12//")
+        runTest("hello:password@go12", "hello:password@go12")
+        runTest("hello:password@go12 lala", "hello:password@go12")
+        runTest("hello.com..", "hello.com.")
+        runTest("a/")
+        runTest("4/5")
+        runTest("concerns/worries")
+        runTest("asdflocalhost aksdjfhads")
+        runTest("/")
+        runTest("////")
+        runTest("hi:")
+        runTest("hi: ")
+        runTest("hi:\n")
+        runTest("testing normal phrase")
+        runTest("testing normal/something phrase")
+        runTest("testing normal: phrase")
+    }
+
+    @Test
+    fun testLongSingleLabelDomain() {
+        runTest("user:password@localhost", "user:password@localhost")
+    }
+
+    @Test
+    fun testShortSingleLabelDomain() {
+        runTest("user:password@go12", "user:password@go12")
+    }
+
     @Test
     fun testIssueUnderscore() {
         runTest("Neomobius_at_mstdn.jp@mostr.pub", "Neomobius_at_mstdn.jp@mostr.pub")
     }
 
+    @Test
+    fun testNostr() {
+        runTest("Check this post nostr:somethingsomething . I think it is really cool", "nostr:somethingsomething")
+    }
+
+    @Test
+    fun testBlossom() {
+        runTest("Check this image blossom:somethingsomething . I think it is really cool", "blossom:somethingsomething")
+    }
+
+    @Test
+    fun testNostrSlashes() {
+        runTest("Check this post nostr://somethingsomething . I think it is really cool", "nostr://somethingsomething")
+    }
+
+    @Test
+    fun testBlossomWithSlashes() {
+        runTest("Check this image blossom://somethingsomething . I think it is really cool", "blossom://somethingsomething")
+    }
+
+    @Test
+    fun testNostr2() {
+        runTest("I saw this on nostr: somethingsomething. I think it is really cool")
+    }
+
+    @Test
+    fun testBlossom2() {
+        runTest("I saw this on blossom: somethingsomething. I think it is really cool")
+    }
+
+    @Test
+    fun testUnsupportedSchema() {
+        runTest("I saw this on hxxp://test.com I think it is really cool")
+    }
+
+    @Test
+    fun testBasicIPv6() {
+        runTest("I saw this on http://[2001:db8:1f70:0:999:de8:7648:6e8] I think it is really cool", "http://[2001:db8:1f70:0:999:de8:7648:6e8]")
+        runTest("I saw this on http://[2001:db8::1]:80 I think it is really cool", "http://[2001:db8::1]:80")
+        runTest("I saw this on http://[2a01:5cc0:1:2::4] I think it is really cool", "http://[2a01:5cc0:1:2::4]")
+        runTest("I saw this on http://[::1]:3000 I think it is really cool", "http://[::1]:3000")
+    }
+
     private fun runTest(
         text: String,
         vararg expected: String?,