nick/amethyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Adds single label domains to parse nostr: urls too

Browse Source
This commit is contained in:
Vitor Pamplona
2026-03-07 10:51:19 -05:00
parent f4d401bc56
commit 109a5e7cd7
6 changed files with 249 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParser.kt
-3
View File
@@ -65,10 +65,7 @@ class UrlParser {
val urlsWithoutScheme = mutableSetOf<String>()
val emails = mutableSetOf<String>()
println("AABBBCC parseValidUrls ${urls.size}")
urls.forEach {
println("AABBBCC Testing ${it.originalUrl}")
if (it.isValidTopLevelDomain()) {
if (it.wroteWithSchema()) {
if (it.isValidLastHostnameChar()) {
commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParserTest.kt
+2 -5
View File
@@ -20,7 +20,6 @@
*/
package com.vitorpamplona.amethyst.commons.richtext
import kotlin.test.Ignore
import kotlin.test.Test
import kotlin.test.assertEquals
@@ -167,7 +166,7 @@ class UrlParserTest {
fun testNostrUrls() =
test(
"nostr:npub1aabbcc",
Urls(),
Urls(withScheme = setOf("nostr:npub1aabbcc")),
)
@Test
@@ -259,7 +258,6 @@ class UrlParserTest {
)
@Test
@Ignore("We need to make this work")
fun testRelayUrl() =
test(
"wss://test.com",
@@ -267,11 +265,10 @@ class UrlParserTest {
)
@Test
@Ignore("We need to make this work")
fun testBech12() =
test(
"nostr:npub1aabbcc",
Urls(withScheme = setOf("wss://test.com")),
Urls(withScheme = setOf("nostr:npub1aabbcc")),
)
@Test
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/UrlMarker.kt
+6
View File
@@ -47,6 +47,12 @@ class UrlMarker {
}
}
fun hasScheme() = schemeIndex >= 0
fun hasPort() = portIndex >= 0
fun hasUsernamePassword() = usernamePasswordIndex >= 0
/**
* @param urlPart The part you want the index of
* @return Returns the index of the part
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/DomainNameReader.kt
+18 -22
View File
@@ -87,14 +87,6 @@ class DomainNameReader(
ReadUserPass,
}
/**
* The interface that gets called for each character that's non-matching (to a valid domain name character) in to count
* the matching quotes and parenthesis correctly.
*/
interface CharacterHandler {
fun addCharacter(character: Char)
}
/**
* Keeps track the number of dots that were found in the domain name.
*/
@@ -136,6 +128,12 @@ class DomainNameReader(
*/
private var zoneIndex = false
var isIpV4 = false
private set
var isIpV6 = false
private set
fun String.isDotPercent() = this == "%2e" || this == "%2E"
/**
@@ -207,14 +205,6 @@ class DomainNameReader(
}
} else if (isAlpha(curr) || curr == '-' || curr.code >= INTERNATIONAL_CHAR_START) {
numeric = false
} else if (!isNumeric(curr)) {
// if its not _numeric and not alphabetical, then restart searching for a domain from this point.
newStart = index + 1
currentLabelLength = 0
topLevelLength = 0
numeric = true
dots = 0
done = true
}
index++
}
@@ -408,6 +398,8 @@ class DomainNameReader(
this[2] == '-' &&
this[3] == '-'
fun labelCount() = dots + (if (currentLabelLength > 0) 1 else 0)
/**
* Checks the current state of this object and returns if the valid state indicates that the
* object has a valid domain name. If it does, it will return append the last character
@@ -440,15 +432,23 @@ class DomainNameReader(
val domainLength: Int =
buffer.length - startDomainName + (if (currentLabelLength > 0) lastDotLength else 0)
val dotCount = dots + (if (currentLabelLength > 0) 1 else 0)
if (domainLength >= MAX_DOMAIN_LENGTH || (dotCount > MAX_NUMBER_LABELS)) {
if (domainLength >= MAX_DOMAIN_LENGTH || dotCount > MAX_NUMBER_LABELS) {
valid = false
} else if (numeric) {
val testDomain = buffer.substring(startDomainName).lowercase()
valid = isValidIpv4(testDomain)
if (valid) {
isIpV4 = true
}
} else if (seenBracket) {
val testDomain = buffer.substring(startDomainName).lowercase()
valid = isValidIpv6(testDomain)
} else if ((currentLabelLength > 0 && dots >= 1) || (dots >= 2 && currentLabelLength == 0)) {
if (valid) {
isIpV6 = true
}
} else if (buffer.isNotEmpty() && buffer.last() == ':') {
valid = false
} else if ((currentLabelLength > 0 && dots >= 1) || (dots >= 2 && currentLabelLength == 0) || (dots == 0)) {
var topStart: Int = buffer.length - topLevelLength
if (currentLabelLength == 0) {
topStart--
@@ -472,10 +472,6 @@ class DomainNameReader(
return validState
}
// Roll back one char if its invalid to handle: "00:41.<br />"
// This gets detected as 41.br otherwise.
reader.goBack()
// return invalid state.
return ReaderNextState.InvalidDomainName
}
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt
+118 -29
View File
@@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.utils.urldetector.Url
import com.vitorpamplona.quartz.utils.urldetector.UrlMarker
import com.vitorpamplona.quartz.utils.urldetector.UrlPart
import kotlin.math.max
import kotlin.text.deleteRange
class UrlDetector(
content: String,
@@ -44,9 +45,9 @@ class UrlDetector(
private var hasScheme = false
/**
* If the first character in the url is a quote, then look for matching quote at the end.
* has Multi-level labels
*/
private var quoteStart = false
private var isSingleLevelLabel = false
/**
* Stores the found urls.
@@ -62,14 +63,7 @@ class UrlDetector(
* The states to use to continue writing or not.
*/
enum class ReadEndState {
/**
* The current url is valid.
*/
ValidUrl,
/**
* The current url is invalid.
*/
InvalidUrl,
}
@@ -95,6 +89,15 @@ class UrlDetector(
// read the next char to process.
when (val curr = reader.read()) {
' ' -> {
// space was found, check if it's a valid single level domain.
if (buffer.isNotEmpty() && hasScheme) {
reader.goBack()
val domain = buffer.substring(length)
if (!readDomainName(domain)) {
readEnd(ReadEndState.InvalidUrl)
}
}
buffer.append(curr)
readEnd(ReadEndState.InvalidUrl)
length = 0
@@ -160,7 +163,7 @@ class UrlDetector(
'/' -> {
// "/" was found, then we either read a scheme, or if we already read a scheme, then
// we are reading a url in the format http://123123123/asdf
if (hasScheme) {
if (hasScheme || buffer.length > 1) {
// we already have the scheme, so then we already read:
// http://something/ <- if something is all numeric then its a valid url.
// OR we are searching for single level domains. We have buffer length > 1 condition
@@ -202,6 +205,13 @@ class UrlDetector(
position = reader.position
}
// check if it's a valid single level domain.
if (buffer.isNotEmpty() && hasScheme) {
if (!readDomainName(buffer.substring(length))) {
readEnd(ReadEndState.InvalidUrl)
}
}
}
/**
@@ -218,7 +228,7 @@ class UrlDetector(
reader.goBack()
// Check buffer length before clearing it; set length to 0 if buffer is empty
if (buffer.length > 0) {
if (buffer.isNotEmpty()) {
buffer.deleteRange(buffer.length - 1, buffer.length)
} else {
length = 0
@@ -237,6 +247,13 @@ class UrlDetector(
} else if (readScheme() && buffer.isNotEmpty()) {
hasScheme = true
length = buffer.length // set length to be right after the scheme
} else if (buffer.isNotEmpty() && reader.canReadChars(1)) {
// takes care of case like hi:
reader.goBack() // unread the ":" so readDomainName can take care of the port
buffer.deleteAt(buffer.length - 1)
if (!readDomainName(buffer.toString())) {
readEnd(ReadEndState.InvalidUrl)
}
} else {
readEnd(ReadEndState.InvalidUrl)
length = 0
@@ -302,6 +319,21 @@ class UrlDetector(
} else if (curr == '[') { // if we're starting to see an ipv6 address
reader.goBack() // unread the '[', so that we can start looking for ipv6
return false
} else if (originalLength > 0 && numSlashes == 0 && CharUtils.isAlpha(curr)) {
// If we had already read something before the : and we are matching regardless of slashes, assume it's a scheme
// Add the slashes to the end of the scheme so it matches what's in the scheme list
val schemeStartIndex = findValidSchemeNoSlashesStartIndex(buffer.toString())
if (schemeStartIndex >= 0) {
buffer.deleteRange(0, schemeStartIndex)
currentUrlMarker.setIndex(UrlPart.SCHEME, 0)
reader.goBack()
return true
} else {
reader.goBack()
return readUserPass(0)
}
// If this didn't match a defined scheme, continue processing as usual
} else if (originalLength > 0 || numSlashes > 0 || !CharUtils.isAlpha(curr)) {
// if it's not a character a-z or A-Z then assume we aren't matching scheme, but instead
// matching username and password.
@@ -321,6 +353,14 @@ class UrlDetector(
.firstOrNull() ?: -1
}
private fun findValidSchemeNoSlashesStartIndex(optionalScheme: String): Int {
val optionalSchemeLowercase = optionalScheme.lowercase()
return VALID_SCHEMES_NO_SLASHES
.filter(optionalSchemeLowercase::endsWith)
.map(optionalSchemeLowercase::lastIndexOf)
.firstOrNull() ?: -1
}
/**
* Reads the input and looks for a username and password.
* Handles:
@@ -390,6 +430,9 @@ class UrlDetector(
// Try to read the dns and act on the response.
val state = reader.readDomainName()
isSingleLevelLabel = reader.labelCount() <= 1 && !reader.isIpV4 && !reader.isIpV6
return when (state) {
DomainNameReader.ReaderNextState.ValidDomainName -> {
readEnd(ReadEndState.ValidUrl)
@@ -481,9 +524,27 @@ class UrlDetector(
while (!reader.eof()) {
// read the next one and remember the length
val curr = reader.read()
// requires at least one number as port to
// better handle http://http://
if (portLen == 0 && isSingleLevelLabel) {
if (!CharUtils.isNumeric(curr)) {
reader.goBack()
currentUrlMarker.unsetIndex(UrlPart.PORT)
return readEnd(ReadEndState.InvalidUrl)
}
}
portLen++
if (curr == '/') {
if (curr == ':') {
// rejects a second port
reader.goBack()
currentUrlMarker.unsetIndex(UrlPart.PORT)
return readEnd(ReadEndState.InvalidUrl)
} else if (curr == '/') {
// continue to read path
buffer.append(curr)
return readPath()
@@ -522,13 +583,27 @@ class UrlDetector(
*/
private fun readPath(): Boolean {
currentUrlMarker.setIndex(UrlPart.PATH, buffer.length - 1)
var endsOnASlash = true
while (!reader.eof()) {
// read the next char
val curr = reader.read()
if (curr == ' ') {
// if end of state and we got here, then the url is valid.
// if end of state and we got here, then the url is valid
// if it is not just a word/word
if (
currentUrlMarker.hasScheme() ||
currentUrlMarker.hasPort() ||
currentUrlMarker.hasUsernamePassword() ||
!isSingleLevelLabel ||
endsOnASlash
) {
return readEnd(ReadEndState.ValidUrl)
} else {
return readEnd(ReadEndState.InvalidUrl)
}
}
// append the char
@@ -542,10 +617,26 @@ class UrlDetector(
// if # read the fragment
return readFragment()
}
endsOnASlash = curr == '/'
}
// end of input then this url is good.
// if end of state and we got here, then the url is valid
// if it is not just a word/word
// no need to check for query and fragments
// here we accept urls that end in /
if (
currentUrlMarker.hasScheme() ||
currentUrlMarker.hasPort() ||
currentUrlMarker.hasUsernamePassword() ||
!isSingleLevelLabel ||
endsOnASlash
) {
return readEnd(ReadEndState.ValidUrl)
} else {
return readEnd(ReadEndState.InvalidUrl)
}
}
/**
@@ -556,12 +647,6 @@ class UrlDetector(
private fun readEnd(state: ReadEndState?): Boolean {
// if the url is valid and greater then 0
if (state == ReadEndState.ValidUrl && buffer.isNotEmpty()) {
// get the last character. if its a quote, cut it off.
val len: Int = buffer.length
if (quoteStart && buffer[len - 1] == '\"') {
buffer.deleteRange(len - 1, len)
}
// Add the url to the list of good urls.
if (buffer.isNotEmpty()) {
currentUrlMarker.originalUrl = buffer.toString()
@@ -573,7 +658,6 @@ class UrlDetector(
buffer.deleteRange(0, buffer.length)
// reset the state of internal objects.
quoteStart = false
hasScheme = false
currentUrlMarker = UrlMarker()
@@ -582,16 +666,21 @@ class UrlDetector(
}
companion object {
private val VALID_SCHEMES: List<String> =
private val VALID_SCHEMES_NO_SLASHES: List<String> =
listOf(
"http://",
"https://",
"ftp://",
"ftps://",
"ws://",
"wss://",
// "nostr:",
// "blossom:",
"http:",
"https:",
"ftp:",
"ftps:",
"ws:",
"wss:",
"nostr:",
"blossom:",
)
private val VALID_SCHEMES =
VALID_SCHEMES_NO_SLASHES.map {
"$it//"
}
}
}
quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt
+103 -1
View File
@@ -21,6 +21,7 @@
package com.vitorpamplona.quartz.utils.urldetector.detection
import com.vitorpamplona.quartz.utils.urldetector.Url
import kotlinx.coroutines.test.runTest
import kotlin.test.Test
import kotlin.test.assertEquals
@@ -134,6 +135,7 @@ class UriDetectionTest {
"Do newlines and tabs break? google.com/hello/\nworld www.yahoo.com\t/stuff/ yahoo.com/\thello news.ycombinator.com\u0000/hello world",
"google.com/hello/",
"www.yahoo.com",
"stuff/",
"yahoo.com/",
"news.ycombinator.com",
)
@@ -335,6 +337,10 @@ class UriDetectionTest {
)
runTest(
"browsers dont support this without a scheme look 0xC00002EB/bobo",
"0xC00002EB/bobo",
)
runTest(
"browsers dont support this without a scheme look test/bobo",
)
}
@@ -350,11 +356,19 @@ class UriDetectionTest {
)
runTest(
"browsers dont support this without a scheme look 030000001353/bobo",
"030000001353/bobo",
)
runTest(
"browsers dont support this without a scheme look 1727123/bobo",
)
}
@Test
fun testUrlWithEmptyPort() {
runTest(
"http://wtfismyip.com:/foo.html",
"http://wtfismyip.com:/foo.html",
)
runTest(
"http://wtfismyip.com://foo.html",
"http://wtfismyip.com://foo.html",
@@ -465,7 +479,9 @@ class UriDetectionTest {
runTest("[b[::7f8e]:55]akjef[::]", "[::7f8e]:55", "[::]")
runTest(
"[bcad::kkkk:aaaa:3dd0[::7f8e]:57b7:34d5]akjef[::]",
"aaaa:3",
"[::7f8e]:57",
"b7:34",
"[::]",
)
}
@@ -550,6 +566,7 @@ class UriDetectionTest {
"Do newlines and tabs break? [::2e80:0:0]/hello/\nworld [::BEEF:ADD:BEEF]\t/stuff/ [AAbb:AAbb:AAbb::]/\thello [::2e80:0:0\u0000]/hello world",
"[::2e80:0:0]/hello/",
"[::BEEF:ADD:BEEF]",
"stuff/",
"[AAbb:AAbb:AAbb::]/",
)
}
@@ -622,17 +639,102 @@ class UriDetectionTest {
val parser = UrlDetector("ftp:example.com")
val found: List<Url> = parser.detect()
for (url in found) {
assertEquals(url.scheme, "https")
assertEquals(url.scheme, "ftp")
// Should be detected as a username now and set to default http://
assertEquals(url.host, "example.com")
}
}
@Test
fun testSingleLevelDomain() {
runTest("http://localhost:9000/lalala hehe", "http://localhost:9000/lalala")
runTest("localhost:9000/lalala hehe", "localhost:9000/lalala")
runTest("http://localhost lasdf", "http://localhost")
runTest("localhost:9000/lalala", "localhost:9000/lalala")
runTest("192.168.1.1/lalala", "192.168.1.1/lalala")
runTest("http://localhost", "http://localhost")
runTest("//localhost", "//localhost")
runTest("asf//localhost")
runTest("hello/", "hello/")
runTest("hello/ ", "hello/")
runTest("hello")
runTest("go/", "go/")
runTest("hello:password@go12//", "hello:password@go12//")
runTest("hello:password@go12", "hello:password@go12")
runTest("hello:password@go12 lala", "hello:password@go12")
runTest("hello.com..", "hello.com.")
runTest("a/")
runTest("4/5")
runTest("concerns/worries")
runTest("asdflocalhost aksdjfhads")
runTest("/")
runTest("////")
runTest("hi:")
runTest("hi: ")
runTest("hi:\n")
runTest("testing normal phrase")
runTest("testing normal/something phrase")
runTest("testing normal: phrase")
}
@Test
fun testLongSingleLabelDomain() {
runTest("user:password@localhost", "user:password@localhost")
}
@Test
fun testShortSingleLabelDomain() {
runTest("user:password@go12", "user:password@go12")
}
@Test
fun testIssueUnderscore() {
runTest("Neomobius_at_mstdn.jp@mostr.pub", "Neomobius_at_mstdn.jp@mostr.pub")
}
@Test
fun testNostr() {
runTest("Check this post nostr:somethingsomething . I think it is really cool", "nostr:somethingsomething")
}
@Test
fun testBlossom() {
runTest("Check this image blossom:somethingsomething . I think it is really cool", "blossom:somethingsomething")
}
@Test
fun testNostrSlashes() {
runTest("Check this post nostr://somethingsomething . I think it is really cool", "nostr://somethingsomething")
}
@Test
fun testBlossomWithSlashes() {
runTest("Check this image blossom://somethingsomething . I think it is really cool", "blossom://somethingsomething")
}
@Test
fun testNostr2() {
runTest("I saw this on nostr: somethingsomething. I think it is really cool")
}
@Test
fun testBlossom2() {
runTest("I saw this on blossom: somethingsomething. I think it is really cool")
}
@Test
fun testUnsupportedSchema() {
runTest("I saw this on hxxp://test.com I think it is really cool")
}
@Test
fun testBasicIPv6() {
runTest("I saw this on http://[2001:db8:1f70:0:999:de8:7648:6e8] I think it is really cool", "http://[2001:db8:1f70:0:999:de8:7648:6e8]")
runTest("I saw this on http://[2001:db8::1]:80 I think it is really cool", "http://[2001:db8::1]:80")
runTest("I saw this on http://[2a01:5cc0:1:2::4] I think it is really cool", "http://[2a01:5cc0:1:2::4]")
runTest("I saw this on http://[::1]:3000 I think it is really cool", "http://[::1]:3000")
}
private fun runTest(
text: String,
vararg expected: String?,