diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParser.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParser.kt index 088c66171..9c8f3e13c 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParser.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParser.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.commons.richtext -import com.vitorpamplona.amethyst.commons.richtext.RichTextParser.Companion.noProtocolUrlValidator import com.vitorpamplona.quartz.utils.urldetector.Url import com.vitorpamplona.quartz.utils.urldetector.detection.UrlDetector @@ -48,7 +47,12 @@ class UrlParser { fun Url.wroteWithSchema(): Boolean = originalUrl.startsWith(scheme) - fun Url.isEmail(): Boolean = originalUrl.contains('@') && path == "/" && query.isEmpty() && fragment.isEmpty() + fun Url.isEmail(): Boolean = + urlMarker.hasUsernamePassword() && + !urlMarker.hasQuery() && + !urlMarker.hasFragment() && + originalUrl.contains('@') && + path == "/" fun Char.isValidLastHostnameChar(): Boolean = (this in 'a'..'z' || this in 'A'..'Z' || this in '0'..'9') @@ -85,12 +89,7 @@ class UrlParser { emails.add(it.value) } } else { - noProtocolUrlValidator.findAll(it.originalUrl).forEach { components -> - val url = components.groups[1]?.value - if (url != null) { - urlsWithoutScheme.add(url) - } - } + urlsWithoutScheme.add(it.originalUrl) } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/Url.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/Url.kt index dbf9858a9..538d9ea32 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/Url.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/Url.kt @@ -35,6 +35,7 @@ package com.vitorpamplona.quartz.utils.urldetector */ class Url( val urlMarker: UrlMarker, + val originalUrl: String, ) { private var _scheme: String? = null private var _username: String? = null @@ -44,7 +45,6 @@ class Url( private var rawPath: String? = null private var _query: String? = null private var _fragment: String? = null - val originalUrl: String = urlMarker.originalUrl override fun toString(): String = this.fullUrl diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarker.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarker.kt index ee28d2122..7a94f90f8 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarker.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarker.kt @@ -28,14 +28,17 @@ class UrlMarker { private var pathIndex = -1 private var queryIndex = -1 private var fragmentIndex = -1 - var originalUrl: String = "" - fun createUrl(): Url = Url(this) + var hasChanged: Boolean = false + private set + + fun createUrl(originalUrl: String): Url = Url(this, originalUrl) fun setIndex( urlPart: UrlPart, index: Int, ) { + hasChanged = true when (urlPart) { UrlPart.SCHEME -> schemeIndex = index UrlPart.USERNAME_PASSWORD -> usernamePasswordIndex = index @@ -53,6 +56,10 @@ class UrlMarker { fun hasUsernamePassword() = usernamePasswordIndex >= 0 + fun hasQuery() = queryIndex >= 0 + + fun hasFragment() = fragmentIndex >= 0 + /** * @param urlPart The part you want the index of * @return Returns the index of the part diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/DomainNameReader.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/DomainNameReader.kt index 48cd4dae1..7704434b0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/DomainNameReader.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/DomainNameReader.kt @@ -166,12 +166,17 @@ class DomainNameReader( var isAllHexSoFar = length > 2 && (currArray[0] == '0' && (currArray[1] == 'x' || currArray[1] == 'X')) + var lastWasAscii = length > 0 && currArray[0].code < INTERNATIONAL_CHAR_START + var index = if (isAllHexSoFar) 2 else 0 var done = false + var isAscii = false while (index < length && !done) { // get the current character and update length counts. val curr = currArray[index] + isAscii = curr.code < INTERNATIONAL_CHAR_START + currentLabelLength++ topLevelLength = currentLabelLength @@ -185,9 +190,7 @@ class DomainNameReader( } else if (curr == '[') { seenBracket = true numeric = false - } else if (curr == '%' && index + 2 < length && isHex(currArray[index + 1]) && - isHex(currArray[index + 2]) - ) { + } else if (curr == '%' && index + 2 < length && isHex(currArray[index + 1]) && isHex(currArray[index + 2])) { // handle url encoded dot if (currArray[index + 1] == '2' && currArray[index + 2] == 'e') { dots++ @@ -203,8 +206,20 @@ class DomainNameReader( isAllHexSoFar = false index-- // backtrack to rerun last character knowing it isn't hex. } - } else if (isAlpha(curr) || curr == '-' || curr.code >= INTERNATIONAL_CHAR_START) { + } else if (isAscii == lastWasAscii && (isAlpha(curr) || curr == '-' || !isAscii)) { + // we don't allow mixed domains: doesn't come here if it changed form ascii to not ascii. numeric = false + lastWasAscii = isAscii + } else if (isAscii != lastWasAscii) { + // if its not _numeric and not alphabetical, then restart searching for a domain from this point. + newStart = index + currentLabelLength = 0 + topLevelLength = 0 + numeric = true + dots = 0 + // done = true + + lastWasAscii = isAscii } index++ } @@ -216,7 +231,9 @@ class DomainNameReader( // make sure the location is not at the end. Otherwise the thing is just invalid. if (newStart < current.length) { - buffer.replaceRange(0, buffer.length, current.substring(newStart)) + buffer.clear() + buffer.append(current.substring(newStart)) + // cut out the previous part, so now the domain name has to be from here. startDomainName = 0 @@ -265,8 +282,16 @@ class DomainNameReader( topLevelLength = currentLabelLength } + var lastWasAscii: Boolean? = null + var isAscii = false + while (!done && !reader.eof()) { val curr: Char = reader.read() + isAscii = curr.code < INTERNATIONAL_CHAR_START + if (lastWasAscii == null) { + lastWasAscii = isAscii + } + if (curr == '/') { // continue by reading the path @@ -314,9 +339,7 @@ class DomainNameReader( return ReaderNextState.InvalidDomainName } } - } else if (seenBracket && (isHex(curr) || curr == ':' || curr == '[' || curr == ']' || curr == '%') && - !seenCompleteBracketSet - ) { // if this is an ipv6 address. + } else if (seenBracket && (isHex(curr) || curr == ':' || curr == '[' || curr == ']' || curr == '%') && !seenCompleteBracketSet) { // if this is an ipv6 address. when (curr) { ':' -> { currentLabelLength = 0 @@ -359,12 +382,18 @@ class DomainNameReader( if (!isAllHexSoFar && !isNumeric(curr)) { numeric = false } - - // append to the states. - buffer.append(curr) - currentLabelLength++ - topLevelLength = currentLabelLength + if (isAscii != lastWasAscii) { + reader.goBack() + done = true + } else { + // append to the states. + buffer.append(curr) + currentLabelLength++ + topLevelLength = currentLabelLength + } } + + lastWasAscii = isAscii } else if (curr == '[' && !seenBracket) { seenBracket = true numeric = false @@ -372,9 +401,7 @@ class DomainNameReader( } else if (curr == '[' && seenCompleteBracketSet) { // Case where [::][ ... reader.goBack() done = true - } else if (curr == '%' && reader.canReadChars(2) && isHex(reader.peekChar(0)) && - isHex(reader.peekChar(1)) - ) { + } else if (curr == '%' && reader.canReadChars(2) && isHex(reader.peekChar(0)) && isHex(reader.peekChar(1))) { // append to the states. buffer.append(curr) buffer.append(reader.read()) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt index 664b20d5e..20698e294 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt @@ -95,11 +95,12 @@ class UrlDetector( val domain = buffer.substring(length) if (!readDomainName(domain)) { readEnd(ReadEndState.InvalidUrl) + } else { + readEnd(ReadEndState.InvalidUrl) } + } else { + readEnd(ReadEndState.InvalidUrl) } - - buffer.append(curr) - readEnd(ReadEndState.InvalidUrl) length = 0 } @@ -649,17 +650,19 @@ class UrlDetector( if (state == ReadEndState.ValidUrl && buffer.isNotEmpty()) { // Add the url to the list of good urls. if (buffer.isNotEmpty()) { - currentUrlMarker.originalUrl = buffer.toString() - urlList.add(currentUrlMarker.createUrl()) + urlList.add(currentUrlMarker.createUrl(buffer.toString())) } } // clear out the buffer. - buffer.deleteRange(0, buffer.length) + buffer.clear() // reset the state of internal objects. hasScheme = false - currentUrlMarker = UrlMarker() + isSingleLevelLabel = false + if (currentUrlMarker.hasChanged) { + currentUrlMarker = UrlMarker() + } // return true if valid. return state == ReadEndState.ValidUrl diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarkerTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarkerTest.kt index e156ded0e..660759b16 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarkerTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarkerTest.kt @@ -37,9 +37,8 @@ class UrlMarkerTest { indices: IntArray, ) { val urlMarker = UrlMarker() - urlMarker.originalUrl = testString urlMarker.setIndices(indices) - val url = urlMarker.createUrl() + val url = urlMarker.createUrl(testString) assertEquals(url.host, host, "host, " + testString) assertEquals(url.path, path, "path, " + testString) assertEquals(url.scheme, scheme, "scheme, " + testString) diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt index 8d89fa81d..b0af22216 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt @@ -735,6 +735,13 @@ class UriDetectionTest { runTest("I saw this on http://[::1]:3000 I think it is really cool", "http://[::1]:3000") } + @Test + fun testNoSchemaUrlMultibyteAscii() { + runTest("ほtest.com", "test.com") + runTest("test.comほ", "test.com") + runTest("ほtest.comほ", "test.com") + } + private fun runTest( text: String, vararg expected: String?,