Adds treatment for the transition between ASCII and non-ASCII in the middle of urls

commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParser.kt

@@ -20,7 +20,6 @@
  */
 package com.vitorpamplona.amethyst.commons.richtext
 
-import com.vitorpamplona.amethyst.commons.richtext.RichTextParser.Companion.noProtocolUrlValidator
 import com.vitorpamplona.quartz.utils.urldetector.Url
 import com.vitorpamplona.quartz.utils.urldetector.detection.UrlDetector
 
@@ -48,7 +47,12 @@ class UrlParser {
 
     fun Url.wroteWithSchema(): Boolean = originalUrl.startsWith(scheme)
 
-    fun Url.isEmail(): Boolean = originalUrl.contains('@') && path == "/" && query.isEmpty() && fragment.isEmpty()
+    fun Url.isEmail(): Boolean =
+        urlMarker.hasUsernamePassword() &&
+            !urlMarker.hasQuery() &&
+            !urlMarker.hasFragment() &&
+            originalUrl.contains('@') &&
+            path == "/"
 
     fun Char.isValidLastHostnameChar(): Boolean = (this in 'a'..'z' || this in 'A'..'Z' || this in '0'..'9')
 
@@ -85,12 +89,7 @@ class UrlParser {
                             emails.add(it.value)
                         }
                     } else {
-                        noProtocolUrlValidator.findAll(it.originalUrl).forEach { components ->
+                        urlsWithoutScheme.add(it.originalUrl)
-                            val url = components.groups[1]?.value
-                            if (url != null) {
-                                urlsWithoutScheme.add(url)
-                            }
-                        }
                     }
                 }
             }

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/Url.kt

@@ -35,6 +35,7 @@ package com.vitorpamplona.quartz.utils.urldetector
  */
 class Url(
     val urlMarker: UrlMarker,
+    val originalUrl: String,
 ) {
     private var _scheme: String? = null
     private var _username: String? = null
@@ -44,7 +45,6 @@ class Url(
     private var rawPath: String? = null
     private var _query: String? = null
     private var _fragment: String? = null
-    val originalUrl: String = urlMarker.originalUrl
 
     override fun toString(): String = this.fullUrl
 

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarker.kt

@@ -28,14 +28,17 @@ class UrlMarker {
     private var pathIndex = -1
     private var queryIndex = -1
     private var fragmentIndex = -1
-    var originalUrl: String = ""
 
-    fun createUrl(): Url = Url(this)
+    var hasChanged: Boolean = false
+        private set
+
+    fun createUrl(originalUrl: String): Url = Url(this, originalUrl)
 
     fun setIndex(
         urlPart: UrlPart,
         index: Int,
     ) {
+        hasChanged = true
         when (urlPart) {
             UrlPart.SCHEME -> schemeIndex = index
             UrlPart.USERNAME_PASSWORD -> usernamePasswordIndex = index
@@ -53,6 +56,10 @@ class UrlMarker {
 
     fun hasUsernamePassword() = usernamePasswordIndex >= 0
 
+    fun hasQuery() = queryIndex >= 0
+
+    fun hasFragment() = fragmentIndex >= 0
+
     /**
      * @param urlPart The part you want the index of
      * @return Returns the index of the part

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/DomainNameReader.kt

@@ -166,12 +166,17 @@ class DomainNameReader(
             var isAllHexSoFar =
                 length > 2 && (currArray[0] == '0' && (currArray[1] == 'x' || currArray[1] == 'X'))
 
+            var lastWasAscii = length > 0 && currArray[0].code < INTERNATIONAL_CHAR_START
+
             var index = if (isAllHexSoFar) 2 else 0
             var done = false
+            var isAscii = false
 
             while (index < length && !done) {
                 // get the current character and update length counts.
                 val curr = currArray[index]
+                isAscii = curr.code < INTERNATIONAL_CHAR_START
+
                 currentLabelLength++
                 topLevelLength = currentLabelLength
 
@@ -185,9 +190,7 @@ class DomainNameReader(
                 } else if (curr == '[') {
                     seenBracket = true
                     numeric = false
-                } else if (curr == '%' && index + 2 < length && isHex(currArray[index + 1]) &&
+                } else if (curr == '%' && index + 2 < length && isHex(currArray[index + 1]) && isHex(currArray[index + 2])) {
-                    isHex(currArray[index + 2])
-                ) {
                     // handle url encoded dot
                     if (currArray[index + 1] == '2' && currArray[index + 2] == 'e') {
                         dots++
@@ -203,8 +206,20 @@ class DomainNameReader(
                         isAllHexSoFar = false
                         index-- // backtrack to rerun last character knowing it isn't hex.
                     }
-                } else if (isAlpha(curr) || curr == '-' || curr.code >= INTERNATIONAL_CHAR_START) {
+                } else if (isAscii == lastWasAscii && (isAlpha(curr) || curr == '-' || !isAscii)) {
+                    // we don't  allow mixed domains: doesn't come here if it changed form ascii to not ascii.
                     numeric = false
+                    lastWasAscii = isAscii
+                } else if (isAscii != lastWasAscii) {
+                    // if its not _numeric and not alphabetical, then restart searching for a domain from this point.
+                    newStart = index
+                    currentLabelLength = 0
+                    topLevelLength = 0
+                    numeric = true
+                    dots = 0
+                    // done = true
+
+                    lastWasAscii = isAscii
                 }
                 index++
             }
@@ -216,7 +231,9 @@ class DomainNameReader(
                 // make sure the location is not at the end. Otherwise the thing is just invalid.
 
                 if (newStart < current.length) {
-                    buffer.replaceRange(0, buffer.length, current.substring(newStart))
+                    buffer.clear()
+                    buffer.append(current.substring(newStart))
+
 
                     // cut out the previous part, so now the domain name has to be from here.
                     startDomainName = 0
@@ -265,8 +282,16 @@ class DomainNameReader(
             topLevelLength = currentLabelLength
         }
 
+        var lastWasAscii: Boolean? = null
+        var isAscii = false
+
         while (!done && !reader.eof()) {
             val curr: Char = reader.read()
+            isAscii = curr.code < INTERNATIONAL_CHAR_START
+            if (lastWasAscii == null) {
+                lastWasAscii = isAscii
+            }
+
 
             if (curr == '/') {
                 // continue by reading the path
@@ -314,9 +339,7 @@ class DomainNameReader(
                         return ReaderNextState.InvalidDomainName
                     }
                 }
-            } else if (seenBracket && (isHex(curr) || curr == ':' || curr == '[' || curr == ']' || curr == '%') &&
+            } else if (seenBracket && (isHex(curr) || curr == ':' || curr == '[' || curr == ']' || curr == '%') && !seenCompleteBracketSet) { // if this is an ipv6 address.
-                !seenCompleteBracketSet
-            ) { // if this is an ipv6 address.
                 when (curr) {
                     ':' -> {
                         currentLabelLength = 0
@@ -359,12 +382,18 @@ class DomainNameReader(
                     if (!isAllHexSoFar && !isNumeric(curr)) {
                         numeric = false
                     }
-
+                    if (isAscii != lastWasAscii) {
-                    // append to the states.
+                        reader.goBack()
-                    buffer.append(curr)
+                        done = true
-                    currentLabelLength++
+                    } else {
-                    topLevelLength = currentLabelLength
+                        // append to the states.
+                        buffer.append(curr)
+                        currentLabelLength++
+                        topLevelLength = currentLabelLength
+                    }
                 }
+
+                lastWasAscii = isAscii
             } else if (curr == '[' && !seenBracket) {
                 seenBracket = true
                 numeric = false
@@ -372,9 +401,7 @@ class DomainNameReader(
             } else if (curr == '[' && seenCompleteBracketSet) { // Case where [::][ ...
                 reader.goBack()
                 done = true
-            } else if (curr == '%' && reader.canReadChars(2) && isHex(reader.peekChar(0)) &&
+            } else if (curr == '%' && reader.canReadChars(2) && isHex(reader.peekChar(0)) && isHex(reader.peekChar(1))) {
-                isHex(reader.peekChar(1))
-            ) {
                 // append to the states.
                 buffer.append(curr)
                 buffer.append(reader.read())

quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt

@@ -95,11 +95,12 @@ class UrlDetector(
                         val domain = buffer.substring(length)
                         if (!readDomainName(domain)) {
                             readEnd(ReadEndState.InvalidUrl)
+                        } else {
+                            readEnd(ReadEndState.InvalidUrl)
                         }
+                    } else {
+                        readEnd(ReadEndState.InvalidUrl)
                     }
-
-                    buffer.append(curr)
-                    readEnd(ReadEndState.InvalidUrl)
                     length = 0
                 }
 
@@ -649,17 +650,19 @@ class UrlDetector(
         if (state == ReadEndState.ValidUrl && buffer.isNotEmpty()) {
             // Add the url to the list of good urls.
             if (buffer.isNotEmpty()) {
-                currentUrlMarker.originalUrl = buffer.toString()
+                urlList.add(currentUrlMarker.createUrl(buffer.toString()))
-                urlList.add(currentUrlMarker.createUrl())
             }
         }
 
         // clear out the buffer.
-        buffer.deleteRange(0, buffer.length)
+        buffer.clear()
 
         // reset the state of internal objects.
         hasScheme = false
-        currentUrlMarker = UrlMarker()
+        isSingleLevelLabel = false
+        if (currentUrlMarker.hasChanged) {
+            currentUrlMarker = UrlMarker()
+        }
 
         // return true if valid.
         return state == ReadEndState.ValidUrl

quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarkerTest.kt

@@ -37,9 +37,8 @@ class UrlMarkerTest {
         indices: IntArray,
     ) {
         val urlMarker = UrlMarker()
-        urlMarker.originalUrl = testString
         urlMarker.setIndices(indices)
-        val url = urlMarker.createUrl()
+        val url = urlMarker.createUrl(testString)
         assertEquals(url.host, host, "host, " + testString)
         assertEquals(url.path, path, "path, " + testString)
         assertEquals(url.scheme, scheme, "scheme, " + testString)

quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt

@@ -735,6 +735,13 @@ class UriDetectionTest {
         runTest("I saw this on http://[::1]:3000 I think it is really cool", "http://[::1]:3000")
     }
 
+    @Test
+    fun testNoSchemaUrlMultibyteAscii() {
+        runTest("ほtest.com", "test.com")
+        runTest("test.comほ", "test.com")
+        runTest("ほtest.comほ", "test.com")
+    }
+
     private fun runTest(
         text: String,
         vararg expected: String?,