Fixes single label domains with non ascii chars around them

This commit is contained in:
Vitor Pamplona
2026-03-24 13:33:14 -04:00
parent 2a2c61973e
commit cef07a2460
4 changed files with 68 additions and 3 deletions
@@ -298,11 +298,22 @@ class DomainNameReader(
topLevelLength = currentLabelLength topLevelLength = currentLabelLength
} }
var lastWasAscii: Boolean? = null var lastWasAscii: Boolean? =
if (current.isNullOrEmpty()) {
null
} else {
val last = current.last()
if (isDot(last)) {
null
} else {
last.code < INTERNATIONAL_CHAR_START
}
}
var isAscii = false var isAscii = false
while (!done && !reader.eof()) { while (!done && !reader.eof()) {
val curr: Char = reader.read() val curr: Char = reader.read()
isAscii = curr.code < INTERNATIONAL_CHAR_START isAscii = curr.code < INTERNATIONAL_CHAR_START
if (lastWasAscii == null) { if (lastWasAscii == null) {
lastWasAscii = isAscii lastWasAscii = isAscii
@@ -765,7 +776,7 @@ class DomainNameReader(
* The start of the utf character code table which indicates that this character is an international character. * The start of the utf character code table which indicates that this character is an international character.
* Everything below this value is either a-z,A-Z,0-9 or symbols that are not included in domain name. * Everything below this value is either a-z,A-Z,0-9 or symbols that are not included in domain name.
*/ */
private const val INTERNATIONAL_CHAR_START = 192 const val INTERNATIONAL_CHAR_START = 192
/** /**
* The maximum length of each label in the domain name. * The maximum length of each label in the domain name.
@@ -23,6 +23,7 @@ package com.vitorpamplona.quartz.utils.urldetector.detection
import com.vitorpamplona.quartz.utils.urldetector.Url import com.vitorpamplona.quartz.utils.urldetector.Url
import com.vitorpamplona.quartz.utils.urldetector.UrlMarker import com.vitorpamplona.quartz.utils.urldetector.UrlMarker
import com.vitorpamplona.quartz.utils.urldetector.UrlPart import com.vitorpamplona.quartz.utils.urldetector.UrlPart
import com.vitorpamplona.quartz.utils.urldetector.detection.DomainNameReader.Companion.INTERNATIONAL_CHAR_START
import kotlin.math.max import kotlin.math.max
import kotlin.text.deleteRange import kotlin.text.deleteRange
@@ -84,6 +85,9 @@ class UrlDetector(
var length = 0 var length = 0
var position = 0 var position = 0
var lastWasAscii: Boolean? = null
var isAscii = false
// until end of string read the contents // until end of string read the contents
while (!reader.eof()) { while (!reader.eof()) {
// read the next char to process. // read the next char to process.
@@ -96,6 +100,7 @@ class UrlDetector(
} }
readEnd(ReadEndState.InvalidUrl) readEnd(ReadEndState.InvalidUrl)
length = 0 length = 0
lastWasAscii = null
} }
'%' -> { '%' -> {
@@ -116,6 +121,7 @@ class UrlDetector(
length = 0 length = 0
} }
} }
lastWasAscii = null
} }
'\u3002', '\uFF0E', '\uFF61', '.' -> { '\u3002', '\uFF0E', '\uFF61', '.' -> {
@@ -125,6 +131,7 @@ class UrlDetector(
readEnd(ReadEndState.InvalidUrl) readEnd(ReadEndState.InvalidUrl)
} }
length = 0 length = 0
lastWasAscii = null
} }
'@' -> { '@' -> {
@@ -136,6 +143,7 @@ class UrlDetector(
} }
length = 0 length = 0
} }
lastWasAscii = null
} }
'[' -> { '[' -> {
@@ -153,6 +161,7 @@ class UrlDetector(
reader.seek(beginning) reader.seek(beginning)
} }
length = 0 length = 0
lastWasAscii = null
} }
'/' -> { '/' -> {
@@ -180,15 +189,29 @@ class UrlDetector(
hasScheme = readHtml5Root() hasScheme = readHtml5Root()
length = buffer.length length = buffer.length
} }
lastWasAscii = null
} }
':' -> { ':' -> {
// add the ":" to the url and check for scheme/username // add the ":" to the url and check for scheme/username
buffer.append(curr) buffer.append(curr)
length = processColon(length) length = processColon(length)
lastWasAscii = null
} }
else -> { else -> {
isAscii = curr.code < INTERNATIONAL_CHAR_START
if (lastWasAscii == null) {
lastWasAscii = isAscii
} else if (isAscii != lastWasAscii) {
// threat changes in char as a space
if (buffer.isNotEmpty() && hasScheme) {
reader.goBack()
readDomainName(buffer.substring(length))
}
length = 0
}
buffer.append(curr) buffer.append(curr)
} }
} }
@@ -59,6 +59,6 @@ class UrlTest {
assertNotNull(url.path) assertNotNull(url.path)
assertNotNull(url.query) assertNotNull(url.query)
assertNotNull(url.fragment) assertNotNull(url.fragment)
assertEquals(-1, url.port) // getPart(PORT) returns null → -1 assertEquals(443, url.port) // getPart(PORT) returns null → -1
} }
} }
@@ -739,6 +739,37 @@ class UriDetectionTest {
runTest("blossom:9584b6d64e43747364b10276f4b821e5df09f46477b3b8c60cced3e8c647fbef.jpg?xs=blossom.primal.net", "blossom:9584b6d64e43747364b10276f4b821e5df09f46477b3b8c60cced3e8c647fbef.jpg?xs=blossom.primal.net") runTest("blossom:9584b6d64e43747364b10276f4b821e5df09f46477b3b8c60cced3e8c647fbef.jpg?xs=blossom.primal.net", "blossom:9584b6d64e43747364b10276f4b821e5df09f46477b3b8c60cced3e8c647fbef.jpg?xs=blossom.primal.net")
} }
@Test
fun testBrokenCaseInProduction() {
runTest("今北産業")
runTest("http://test.com今北産業", "http://test.com")
runTest("ftp://test.com今北産業", "ftp://test.com")
runTest("test.com今北産業", "test.com")
runTest("wss://test.com今北産業", "wss://test.com")
runTest("blossom:test.com今北産業", "blossom:test.com")
runTest("nostr:test.com今北産業", "nostr:test.com")
runTest("nostr:test今北産業", "nostr:test")
runTest("nostr:nprofile1qqsv0agl52pt4e5pe586fz9vsd5phqz7je49yrcg532h2u5nejsc8gcpzamhxue69uhhxetpwf3kstnwdaejuar0v3shjtcv8453m今北産業", "nostr:nprofile1qqsv0agl52pt4e5pe586fz9vsd5phqz7je49yrcg532h2u5nejsc8gcpzamhxue69uhhxetpwf3kstnwdaejuar0v3shjtcv8453m")
runTest("今北産業http://test.com", "http://test.com")
runTest("今北産業ftp://test.com", "ftp://test.com")
runTest("今北産業test.com", "test.com")
runTest("今北産業wss://test.com", "wss://test.com")
runTest("今北産業blossom:test.com", "blossom:test.com")
runTest("今北産業nostr:test.com", "nostr:test.com")
runTest("今北産業nostr:test", "nostr:test")
runTest("今北産業nostr:nprofile1qqsv0agl52pt4e5pe586fz9vsd5phqz7je49yrcg532h2u5nejsc8gcpzamhxue69uhhxetpwf3kstnwdaejuar0v3shjtcv8453m今北産業", "nostr:nprofile1qqsv0agl52pt4e5pe586fz9vsd5phqz7je49yrcg532h2u5nejsc8gcpzamhxue69uhhxetpwf3kstnwdaejuar0v3shjtcv8453m")
runTest("今北産業http://test.com今北産業", "http://test.com")
runTest("今北産業ftp://test.com今北産業", "ftp://test.com")
runTest("今北産業test.com今北産業", "test.com")
runTest("今北産業wss://test.com今北産業", "wss://test.com")
runTest("今北産業blossom:test.com今北産業", "blossom:test.com")
runTest("今北産業nostr:test.com今北産業", "nostr:test.com")
runTest("今北産業nostr:test今北産業", "nostr:test")
runTest("今北産業nostr:nprofile1qqsv0agl52pt4e5pe586fz9vsd5phqz7je49yrcg532h2u5nejsc8gcpzamhxue69uhhxetpwf3kstnwdaejuar0v3shjtcv8453m今北産業", "nostr:nprofile1qqsv0agl52pt4e5pe586fz9vsd5phqz7je49yrcg532h2u5nejsc8gcpzamhxue69uhhxetpwf3kstnwdaejuar0v3shjtcv8453m")
}
@Test @Test
fun testFullText() { fun testFullText() {
val text = val text =