Switches to our own version of the Url Detector

This commit is contained in:
Vitor Pamplona
2026-03-06 10:58:59 -05:00
parent 13f401df8c
commit f4d401bc56
32 changed files with 3794 additions and 156 deletions
+10
View File
@@ -0,0 +1,10 @@
# Default ignored files
/shelf/
/workspace.xml
# Ignored default folder with query files
/queries/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
# Editor-based HTTP Client requests
/httpRequests/
-3
View File
@@ -161,9 +161,6 @@ kotlin {
// Performant Parser of JSONs into Events
api(libs.jackson.module.kotlin)
// Parses URLs from Text:
api(libs.url.detector)
// Websockets API
implementation(libs.okhttp)
implementation(libs.okhttpCoroutines)
@@ -0,0 +1,248 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector
/**
* Creating own Uri class since java.net.Uri would throw parsing exceptions
* for URL's considered ok by browsers.
*
* Also to avoid further conflict, this does stuff that the normal Uri object doesn't do:
* - Converts http://google.com/a/b/.//./../c to http://google.com/a/c
* - Decodes repeatedly so that http://host/%2525252525252525 becomes http://host/%25 while normal decoders
* would make it http://host/%25252525252525 (one less 25)
* - Removes tabs and new lines: http://www.google.com/foo\tbar\rbaz\n2 becomes "http://www.google.com/foobarbaz2"
* - Converts IP addresses: http://3279880203/blah becomes http://195.127.0.11/blah
* - Strips fragments (anything after #)
*
*/
class Url(
val urlMarker: UrlMarker,
) {
private var _scheme: String? = null
private var _username: String? = null
private var _password: String? = null
private var rawHost: String? = null
private var _port = 0
private var rawPath: String? = null
private var _query: String? = null
private var _fragment: String? = null
val originalUrl: String = urlMarker.originalUrl
override fun toString(): String = this.fullUrl
/**
* Note that this includes the fragment
* @return Formats the url to: [scheme]://[username]:[password]@[host]:[port]/[path]?[query]#[fragment]
*/
val fullUrl: String
get() = this.fullUrlWithoutFragment + this.fragment
/**
*
* @return Formats the url to: [scheme]://[username]:[password]@[host]:[port]/[path]?[query]
*/
val fullUrlWithoutFragment: String
get() {
val url = StringBuilder()
if (this.scheme.isNotEmpty()) {
url.append(this.scheme)
url.append(":")
}
url.append("//")
if (this.username.isNotEmpty()) {
url.append(this.username)
if (this.password.isNotEmpty()) {
url.append(":")
url.append(this.password)
}
url.append("@")
}
url.append(this.host)
if (this.port > 0 && this.port != SCHEME_PORT_MAP[this.scheme]) {
url.append(":")
url.append(this.port)
}
url.append(this.path)
url.append(this.query)
return url.toString()
}
val scheme: String
get() {
if (_scheme == null) {
if (exists(UrlPart.SCHEME)) {
_scheme = getPart(UrlPart.SCHEME)
val index = _scheme!!.indexOf(":")
if (index != -1) {
_scheme = _scheme!!.substring(0, index)
}
} else if (!originalUrl.startsWith("//")) {
_scheme =
DEFAULT_SCHEME
}
}
return _scheme ?: ""
}
val username: String
get() {
if (_username == null) {
populateUsernamePassword()
}
return _username ?: ""
}
val password: String
get() {
if (_password == null) {
populateUsernamePassword()
}
return _password ?: ""
}
val host: String
get() {
if (this.rawHost == null) {
this.rawHost = getPart(UrlPart.HOST)
if (exists(UrlPart.PORT)) {
this.rawHost =
rawHost?.let {
it.substring(0, it.length - 1)
}
}
}
return this.rawHost!!
}
/**
* port = 0 means it hasn't been set yet. port = -1 means there is no port
*/
val port: Int
get() {
if (_port == 0) {
val portString =
getPart(UrlPart.PORT)
if (!portString.isNullOrEmpty()) {
_port = portString.toIntOrNull() ?: -1
} else {
_port = SCHEME_PORT_MAP[this.scheme] ?: -1
}
}
return _port
}
val path: String?
get() {
if (this.rawPath == null) {
this.rawPath =
if (exists(UrlPart.PATH)) {
getPart(
UrlPart.PATH,
)
} else {
"/"
}
}
return this.rawPath
}
val query: String
get() {
if (_query == null) {
_query = getPart(UrlPart.QUERY)
}
return _query ?: ""
}
val fragment: String
get() {
if (_fragment == null) {
_fragment = getPart(UrlPart.FRAGMENT)
}
return _fragment ?: ""
}
private fun populateUsernamePassword() {
val usernamePassword = getPart(UrlPart.USERNAME_PASSWORD)
if (usernamePassword != null) {
val usernamePasswordParts: List<String> =
usernamePassword.substring(0, usernamePassword.length - 1).split(":")
if (usernamePasswordParts.size == 1) {
_username = usernamePasswordParts[0]
} else if (usernamePasswordParts.size == 2) {
_username = usernamePasswordParts[0]
_password = usernamePasswordParts[1]
}
}
}
/**
* @param urlPart The url part we are checking for existence
* @return Returns true if the part exists.
*/
private fun exists(urlPart: UrlPart?): Boolean = urlPart != null && urlMarker.indexOf(urlPart) >= 0
/**
* For example, in http://yahoo.com/lala/, nextExistingPart(UrlPart.HOST) would return UrlPart.PATH
* @param urlPart The current url part
* @return Returns the next part; if there is no existing next part, it returns null
*/
private fun nextExistingPart(urlPart: UrlPart): UrlPart? {
val nextPart = urlPart.nextPart
if (exists(nextPart)) {
return nextPart
} else if (nextPart == null) {
return null
} else {
return nextExistingPart(nextPart)
}
}
/**
* @param part The part that we want. Ex: host, path
*/
private fun getPart(part: UrlPart): String? {
if (!exists(part)) {
return null
}
val nextPart = nextExistingPart(part)
return if (nextPart == null) {
originalUrl.substring(urlMarker.indexOf(part))
} else {
originalUrl.substring(urlMarker.indexOf(part), urlMarker.indexOf(nextPart))
}
}
companion object {
private const val DEFAULT_SCHEME = "https"
private val SCHEME_PORT_MAP: Map<String, Int> =
mapOf(
"http" to 80,
"https" to 443,
"ftp" to 21,
)
}
}
@@ -0,0 +1,84 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector
class UrlMarker {
private var schemeIndex = -1
private var usernamePasswordIndex = -1
private var hostIndex = -1
private var portIndex = -1
private var pathIndex = -1
private var queryIndex = -1
private var fragmentIndex = -1
var originalUrl: String = ""
fun createUrl(): Url = Url(this)
fun setIndex(
urlPart: UrlPart,
index: Int,
) {
when (urlPart) {
UrlPart.SCHEME -> schemeIndex = index
UrlPart.USERNAME_PASSWORD -> usernamePasswordIndex = index
UrlPart.HOST -> hostIndex = index
UrlPart.PORT -> portIndex = index
UrlPart.PATH -> pathIndex = index
UrlPart.QUERY -> queryIndex = index
UrlPart.FRAGMENT -> fragmentIndex = index
}
}
/**
* @param urlPart The part you want the index of
* @return Returns the index of the part
*/
fun indexOf(urlPart: UrlPart): Int =
when (urlPart) {
UrlPart.SCHEME -> schemeIndex
UrlPart.USERNAME_PASSWORD -> usernamePasswordIndex
UrlPart.HOST -> hostIndex
UrlPart.PORT -> portIndex
UrlPart.PATH -> pathIndex
UrlPart.QUERY -> queryIndex
UrlPart.FRAGMENT -> fragmentIndex
}
fun unsetIndex(urlPart: UrlPart) {
setIndex(urlPart, -1)
}
/**
* This is used in TestUrlMarker to set indices more easily.
* @param indices array of indices of size 7
*/
fun setIndices(indices: IntArray): UrlMarker {
require(indices.size == 7) { "Malformed index array." }
setIndex(UrlPart.SCHEME, indices[0])
setIndex(UrlPart.USERNAME_PASSWORD, indices[1])
setIndex(UrlPart.HOST, indices[2])
setIndex(UrlPart.PORT, indices[3])
setIndex(UrlPart.PATH, indices[4])
setIndex(UrlPart.QUERY, indices[5])
setIndex(UrlPart.FRAGMENT, indices[6])
return this
}
}
@@ -0,0 +1,36 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector
enum class UrlPart(
nextPart: UrlPart?,
) {
FRAGMENT(null),
QUERY(FRAGMENT),
PATH(QUERY),
PORT(PATH),
HOST(PORT),
USERNAME_PASSWORD(HOST),
SCHEME(USERNAME_PASSWORD),
;
val nextPart: UrlPart? = nextPart
}
@@ -0,0 +1,86 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector.detection
object CharUtils {
/**
* Checks if character is a valid hex character.
*/
fun isHex(a: Char): Boolean = (a in '0'..'9') || (a in 'a'..'f') || (a in 'A'..'F')
/**
* Checks if character is a valid alphabetic character.
*/
fun isAlpha(a: Char): Boolean = ((a in 'a'..'z') || (a in 'A'..'Z'))
/**
* Checks if character is a valid numeric character.
*/
fun isNumeric(a: Char): Boolean = a in '0'..'9'
/**
* Checks if character is a valid alphanumeric character.
*/
fun isAlphaNumeric(a: Char): Boolean = isAlpha(a) || isNumeric(a)
/**
* Checks if character is a valid unreserved character. This is defined by the RFC 3986 ABNF
*/
fun isUnreserved(a: Char): Boolean = isAlphaNumeric(a) || a == '-' || a == '.' || a == '_' || a == '~'
/**
* Checks if character is a dot. Heres the doc:
* http://docs.oracle.com/javase/6/docs/api/java/net/IDN.html#toASCII%28java.lang.String,%20int%29
*/
fun isDot(a: Char): Boolean = (a == '.' || a == '\u3002' || a == '\uFF0E' || a == '\uFF61')
fun isWhiteSpace(a: Char): Boolean = (a == '\n' || a == '\t' || a == '\r' || a == ' ')
/**
* Splits a string without the use of a regex, which could split either by isDot() or %2e
* @param input the input string that will be split by dot
* @return an array of strings that is a partition of the original string split by dot
*/
fun splitByDot(input: String): List<String> {
val splitList = ArrayList<String>()
val section = StringBuilder()
if (input.isEmpty()) {
return listOf("")
}
val reader = InputTextReader(input)
while (!reader.eof()) {
val curr: Char = reader.read()
if (isDot(curr)) {
splitList.add(section.toString())
section.setLength(0)
} else if (curr == '%' && (reader.peekEquals("2e") || reader.peekEquals("2E"))) {
reader.read()
reader.read() // advance past the 2e
splitList.add(section.toString())
section.setLength(0)
} else {
section.append(curr)
}
}
splitList.add(section.toString())
return splitList
}
}
@@ -0,0 +1,760 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector.detection
import com.vitorpamplona.quartz.utils.urldetector.detection.CharUtils.isAlpha
import com.vitorpamplona.quartz.utils.urldetector.detection.CharUtils.isAlphaNumeric
import com.vitorpamplona.quartz.utils.urldetector.detection.CharUtils.isDot
import com.vitorpamplona.quartz.utils.urldetector.detection.CharUtils.isHex
import com.vitorpamplona.quartz.utils.urldetector.detection.CharUtils.isNumeric
import com.vitorpamplona.quartz.utils.urldetector.detection.CharUtils.isUnreserved
import com.vitorpamplona.quartz.utils.urldetector.detection.CharUtils.splitByDot
import kotlin.math.max
import kotlin.math.min
/**
* The domain name reader reads input from a InputTextReader and validates if the content being read is a valid domain name.
* After a domain name is read, the returning status is what to do next. If the domain is valid but a specific character is found,
* the next state will be to read another part for the rest of the url. For example, if a "?" is found at the end and the
* domain is valid, the return state will be to read a query string.
*/
class DomainNameReader(
val reader: InputTextReader,
/**
* The currently written string buffer.
*/
val buffer: StringBuilder,
/**
* The domain name started with a partial domain name found. This is the original string of the domain name only.
*/
val current: String?,
) {
/**
* This is the final return state of reading a domain name.
*/
enum class ReaderNextState {
/**
* Trying to read the domain name caused it to be invalid.
*/
InvalidDomainName,
/**
* The domain name is found to be valid.
*/
ValidDomainName,
/**
* Finished reading, next step should be to read the fragment.
*/
ReadFragment,
/**
* Finished reading, next step should be to read the path.
*/
ReadPath,
/**
* Finished reading, next step should be to read the port.
*/
ReadPort,
/**
* Finished reading, next step should be to read the query string.
*/
ReadQueryString,
/**
* This was actually not a domain at all.
*/
ReadUserPass,
}
/**
* The interface that gets called for each character that's non-matching (to a valid domain name character) in to count
* the matching quotes and parenthesis correctly.
*/
interface CharacterHandler {
fun addCharacter(character: Char)
}
/**
* Keeps track the number of dots that were found in the domain name.
*/
private var dots = 0
/**
* Keeps track of the number of characters since the last "."
*/
private var currentLabelLength = 0
/**
* Keeps track of the number of characters in the top level domain.
*/
private var topLevelLength = 0
/**
* Keeps track where the domain name started. This is non zero if the buffer starts with
* http://username:password@...
*/
private var startDomainName = 0
/**
* Keeps track if the entire domain name is numeric.
*/
private var numeric = false
/**
* Keeps track if we are seeing an ipv6 type address.
*/
private var seenBracket = false
/**
* Keeps track if we have seen a full bracket set "[....]"; used for ipv6 type address.
*/
private var seenCompleteBracketSet = false
/**
* Keeps track if we have a zone index in the ipv6 address.
*/
private var zoneIndex = false
fun String.isDotPercent() = this == "%2e" || this == "%2E"
/**
* Reads and parses the current string to make sure the domain name started where it was supposed to,
* and the current domain name is correct.
* @return The next state to use after reading the current.
*/
private fun readCurrent(): ReaderNextState {
if (current != null) {
// Handles the case where the string is ".hello"
if (current.length == 1 && isDot(current[0])) {
return ReaderNextState.InvalidDomainName
} else if (current.length == 3 && current.isDotPercent()) {
return ReaderNextState.InvalidDomainName
}
// The location where the domain name started.
startDomainName = buffer.length - current.length
// flag that the domain is currently all numbers and/or dots.
numeric = true
// If an invalid char is found, we can just restart the domain from there.
var newStart = 0
val currArray = current.toCharArray()
val length = currArray.size
// hex special case
var isAllHexSoFar =
length > 2 && (currArray[0] == '0' && (currArray[1] == 'x' || currArray[1] == 'X'))
var index = if (isAllHexSoFar) 2 else 0
var done = false
while (index < length && !done) {
// get the current character and update length counts.
val curr = currArray[index]
currentLabelLength++
topLevelLength = currentLabelLength
// Is the length of the last part > 64 (plus one since we just incremented)
if (currentLabelLength > MAX_LABEL_LENGTH) {
return ReaderNextState.InvalidDomainName
} else if (isDot(curr)) {
// found a dot. Increment dot count, and reset last length
dots++
currentLabelLength = 0
} else if (curr == '[') {
seenBracket = true
numeric = false
} else if (curr == '%' && index + 2 < length && isHex(currArray[index + 1]) &&
isHex(currArray[index + 2])
) {
// handle url encoded dot
if (currArray[index + 1] == '2' && currArray[index + 2] == 'e') {
dots++
currentLabelLength = 0
} else {
numeric = false
}
index += 2
} else if (isAllHexSoFar) {
// if it's a valid character in the domain that is not numeric
if (!isHex(curr)) {
numeric = false
isAllHexSoFar = false
index-- // backtrack to rerun last character knowing it isn't hex.
}
} else if (isAlpha(curr) || curr == '-' || curr.code >= INTERNATIONAL_CHAR_START) {
numeric = false
} else if (!isNumeric(curr)) {
// if its not _numeric and not alphabetical, then restart searching for a domain from this point.
newStart = index + 1
currentLabelLength = 0
topLevelLength = 0
numeric = true
dots = 0
done = true
}
index++
}
// An invalid character for the domain was found somewhere in the current buffer.
// cut the first part of the domain out. For example:
// http://asdf%asdf.google.com <- asdf.google.com is still valid, so restart from the %
if (newStart > 0) {
// make sure the location is not at the end. Otherwise the thing is just invalid.
if (newStart < current.length) {
buffer.replaceRange(0, buffer.length, current.substring(newStart))
// cut out the previous part, so now the domain name has to be from here.
startDomainName = 0
}
// now after cutting if the buffer is just "." newStart > current (last character in current is invalid)
if (newStart >= current.length || buffer.toString() == ".") {
return ReaderNextState.InvalidDomainName
}
}
} else {
startDomainName = buffer.length
}
// all else is good, return OK
return ReaderNextState.ValidDomainName
}
/**
* Reads the Dns and returns the next state the state machine should take in throwing this out, or continue processing
* if this is a valid domain name.
* @return The next state to take.
*/
fun readDomainName(): ReaderNextState? {
// Read the current, and if its bad, just return.
if (readCurrent() == ReaderNextState.InvalidDomainName) {
return ReaderNextState.InvalidDomainName
}
// while not done and not end of string keep reading.
var done = false
// If this is the first domain part, check if it's ip address in is hexa
// similar to what is done on 'readCurrent' method
val isAllHexSoFar =
(current == null || current == "") &&
reader.canReadChars(3) &&
(reader.peekEquals("0x") || reader.peekEquals("0X"))
if (isAllHexSoFar) {
// Append hexa radix symbol characters (0x)
buffer.append(reader.read())
buffer.append(reader.read())
currentLabelLength += 2
topLevelLength = currentLabelLength
}
while (!done && !reader.eof()) {
val curr: Char = reader.read()
if (curr == '/') {
// continue by reading the path
return checkDomainNameValid(ReaderNextState.ReadPath, curr)
} else if (curr == ':' && (!seenBracket || seenCompleteBracketSet)) {
// Don't check for a port if it's in the middle of an ipv6 address
// continue by reading the port.
return checkDomainNameValid(ReaderNextState.ReadPort, curr)
} else if (curr == '?') {
// continue by reading the query string
return checkDomainNameValid(ReaderNextState.ReadQueryString, curr)
} else if (curr == '#') {
// continue by reading the fragment
return checkDomainNameValid(ReaderNextState.ReadFragment, curr)
} else if (curr == '@') {
// this may not have been a domain after all, but rather a username/password instead
reader.goBack()
return ReaderNextState.ReadUserPass
} else if (isDot(curr) || (curr == '%' && (reader.peekEquals("2e") || reader.peekEquals("2E")))) {
// if the current character is a dot or a urlEncodedDot
// handles the case: hello..
if (currentLabelLength < 1) {
done = true
} else {
// append the "." to the domain name
buffer.append(curr)
// if it was not a normal dot, then it is url encoded
// read the next two chars, which are the hex representation
if (!isDot(curr)) {
buffer.append(reader.read())
buffer.append(reader.read())
}
// increment the dots only if it's not part of the zone index and reset the last length.
if (!zoneIndex) {
dots++
currentLabelLength = 0
}
// if the length of the last section is longer than or equal to 64, it's too long to be a valid domain
if (currentLabelLength >= MAX_LABEL_LENGTH) {
return ReaderNextState.InvalidDomainName
}
}
} else if (seenBracket && (isHex(curr) || curr == ':' || curr == '[' || curr == ']' || curr == '%') &&
!seenCompleteBracketSet
) { // if this is an ipv6 address.
when (curr) {
':' -> {
currentLabelLength = 0
}
'[' -> {
// if we read another '[', we need to restart by re-reading from this bracket instead.
reader.goBack()
return ReaderNextState.InvalidDomainName
}
']' -> {
seenCompleteBracketSet =
true // means that we already have a complete ipv6 address.
zoneIndex =
false // set this back off so that we can keep counting dots after ipv6 is over.
}
'%' -> {
zoneIndex = true
}
else -> {
currentLabelLength++
}
}
numeric = false
buffer.append(curr)
} else if (isAlphaNumeric(curr) || curr == '-' || curr.code >= INTERNATIONAL_CHAR_START) {
// Valid domain name character. Either a-z, A-Z, 0-9, -, or international character
if (seenCompleteBracketSet) {
// covers case of [fe80::]www.google.com
reader.goBack()
done = true
} else {
if (isAllHexSoFar && !isHex(curr)) {
numeric = false
}
// if its not numeric, remember that;
if (!isAllHexSoFar && !isNumeric(curr)) {
numeric = false
}
// append to the states.
buffer.append(curr)
currentLabelLength++
topLevelLength = currentLabelLength
}
} else if (curr == '[' && !seenBracket) {
seenBracket = true
numeric = false
buffer.append(curr)
} else if (curr == '[' && seenCompleteBracketSet) { // Case where [::][ ...
reader.goBack()
done = true
} else if (curr == '%' && reader.canReadChars(2) && isHex(reader.peekChar(0)) &&
isHex(reader.peekChar(1))
) {
// append to the states.
buffer.append(curr)
buffer.append(reader.read())
buffer.append(reader.read())
currentLabelLength += 3
topLevelLength = currentLabelLength
} else {
// invalid character, we are done.
done = true
}
}
// Check the domain name to make sure its ok.
return checkDomainNameValid(ReaderNextState.ValidDomainName, null)
}
fun String.isXn() =
this.length > 3 &&
(this[0] == 'x' || this[0] == 'X') &&
(this[1] == 'n' || this[1] == 'N') &&
this[2] == '-' &&
this[3] == '-'
/**
* Checks the current state of this object and returns if the valid state indicates that the
* object has a valid domain name. If it does, it will return append the last character
* and return the validState specified.
* @param validState The state to return if this check indicates that the dns is ok.
* @param lastChar The last character to add if the domain is ok.
* @return The validState if the domain is valid, else ReaderNextState.InvalidDomainName
*/
private fun checkDomainNameValid(
validState: ReaderNextState?,
lastChar: Char?,
): ReaderNextState? {
var valid = false
// Max domain length is 255 which includes the trailing "."
// most of the time this is not included in the url.
// If the _currentLabelLength is not 0 then the last "." is not included so add it.
// Same with number of labels (or dots including the last)
val lastDotLength =
if (buffer.length > 3 &&
buffer[buffer.length - 3] == '%' &&
buffer[buffer.length - 2] == '2' &&
(buffer[buffer.length - 1] == 'e' || buffer[buffer.length - 1] == 'E')
) {
3
} else {
1
}
val domainLength: Int =
buffer.length - startDomainName + (if (currentLabelLength > 0) lastDotLength else 0)
val dotCount = dots + (if (currentLabelLength > 0) 1 else 0)
if (domainLength >= MAX_DOMAIN_LENGTH || (dotCount > MAX_NUMBER_LABELS)) {
valid = false
} else if (numeric) {
val testDomain = buffer.substring(startDomainName).lowercase()
valid = isValidIpv4(testDomain)
} else if (seenBracket) {
val testDomain = buffer.substring(startDomainName).lowercase()
valid = isValidIpv6(testDomain)
} else if ((currentLabelLength > 0 && dots >= 1) || (dots >= 2 && currentLabelLength == 0)) {
var topStart: Int = buffer.length - topLevelLength
if (currentLabelLength == 0) {
topStart--
}
topStart = max(topStart, 0)
// get the first 4 characters of the top level domain
val topLevelStart =
buffer.substring(topStart, topStart + min(4, buffer.length - topStart))
// There is no size restriction if the top level domain is international (starts with "xn--")
valid =
((topLevelStart.isXn() || (topLevelLength in MIN_TOP_LEVEL_DOMAIN..MAX_TOP_LEVEL_DOMAIN)))
}
if (valid) {
// if it's valid, add the last character (if specified) and return the valid state.
if (lastChar != null) {
buffer.append(lastChar)
}
return validState
}
// Roll back one char if its invalid to handle: "00:41.<br />"
// This gets detected as 41.br otherwise.
reader.goBack()
// return invalid state.
return ReaderNextState.InvalidDomainName
}
/**
* Handles Hexadecimal, octal, decimal, dotted decimal, dotted hex, dotted octal.
* @param testDomain the string we're testing
* @return Returns true if it's a valid ipv4 address
*/
private fun isValidIpv4(testDomain: String): Boolean {
var valid = false
val length: Int = testDomain.length
if (length > 0) {
// handling format without dots. Ex: http://2123123123123/path/a, http://0x8242343/aksdjf
if (dots == 0) {
try {
val value: Long
if (length > 2 && testDomain[0] == '0' && testDomain[1] == 'x') { // hex
// digit must be within ['0', '9'] or ['A', 'F'] or ['a', 'f']
for (c in 2..<length) {
val d: Char = testDomain[c]
if ((d < '0' || (d in ':'..<'A') || (d in 'G'..<'a') || d > 'f')) {
return false
}
}
value = testDomain.substring(2).toLong(16)
} else if (testDomain[0] == '0') { // octal
// digit must be within ['0', '7']
for (c in 1..<length) {
val d: Char = testDomain[c]
if (d !in '0'..'7') {
return false
}
}
value = testDomain.substring(1).toLong(8)
} else { // decimal
// digit must be within ['0', '9']
for (c in 0..<length) {
val d: Char = testDomain[c]
if (d !in '0'..'9') {
return false
}
}
value = testDomain.toLong()
}
valid = value in MIN_NUMERIC_DOMAIN_VALUE..MAX_NUMERIC_DOMAIN_VALUE
} catch (_: NumberFormatException) {
valid = false
}
} else if (dots == 3) {
// Dotted decimal/hex/octal format
val parts: List<String> = splitByDot(testDomain)
valid = true
// check each part of the ip and make sure its valid.
var i = 0
while (i < parts.size && valid) {
val part = parts[i]
val partLen: Int = part.length
if (partLen > 0) {
val parsedNum: String
val base: Int
if (partLen > 2 && part[0] == '0' && part[1] == 'x') { // dotted hex
// digit must be within ['0', '9'] or ['A', 'F'] or ['a', 'f']
for (c in 2..<partLen) {
val d: Char = part[c]
if ((d < '0' || (d in ':'..<'A') || (d in 'G'..<'a') || d > 'f')) {
return false
}
}
parsedNum = part.substring(2)
base = 16
} else if (part[0] == '0') { // dotted octal
// digit must be within ['0', '7']
for (c in 1..<partLen) {
val d: Char = part[c]
if (d !in '0'..'7') {
return false
}
}
parsedNum = part.substring(1)
base = 8
} else { // dotted decimal
// digit must be within ['0', '9']
for (c in 0..<partLen) {
val d: Char = part[c]
if (d !in '0'..'9') {
return false
}
}
parsedNum = part
base = 10
}
val section =
if (parsedNum.isEmpty()) {
0
} else {
try {
parsedNum.toInt(base)
} catch (_: NumberFormatException) {
return false
}
}
if (section !in MIN_IP_PART..MAX_IP_PART) {
valid = false
}
} else {
valid = false
}
i++
}
}
}
return valid
}
/**
* Sees that there's an open "[", and is now checking for ":"'s and stopping when there is a ']' or invalid character.
* Handles ipv4 formatted ipv6 addresses, zone indices, truncated notation.
* @return Returns true if it is a valid ipv6 address
*/
private fun isValidIpv6(testDomain: String): Boolean {
val domainArray = testDomain.toCharArray()
// Return false if we don't see [....]
// or if we only have '[]'
// or if we detect [:8000: ...]; only [::8000: ...] is okay
if (
domainArray.size < 3 ||
domainArray[domainArray.size - 1] != ']' ||
domainArray[0] != '[' ||
(domainArray[1] == ':' && domainArray[2] != ':')
) {
return false
}
var numSections = 1
var hexDigits = 0
var prevChar = 0.toChar()
// used to check ipv4 addresses at the end of ipv6 addresses.
val lastSection = StringBuilder()
var hexSection = true
// If we see a '%'. Example: http://[::ffff:0xC0.0x00.0x02.0xEB%251]
var zoneIndiceMode = false
// If doubleColonFlag is true, that means we've already seen one "::"; we're not allowed to have more than one.
var doubleColonFlag = false
var index = 0
while (index < domainArray.size) {
when (domainArray[index]) {
'[' -> {}
'%', ']' -> {
var out = false
if (domainArray[index] == '%') {
// see if there's a urlencoded dot
if (domainArray.size - index >= 2 && domainArray[index + 1] == '2' && domainArray[index + 2] == 'e') {
lastSection.append("%2e")
index += 2
hexSection = false
out = true
}
if (!out) zoneIndiceMode = true
}
if (!out) {
if (!hexSection && (!zoneIndiceMode || domainArray[index] == '%')) {
if (isValidIpv4(lastSection.toString())) {
numSections++ // ipv4 takes up 2 sections.
} else {
return false
}
}
}
}
':' -> {
if (prevChar == ':') {
if (doubleColonFlag) { // only allowed to have one "::" in an ipv6 address.
return false
}
doubleColonFlag = true
}
// This means that we reached invalid characters in the previous section
if (!hexSection) {
return false
}
hexSection = true // reset hex to true
hexDigits = 0 // reset count for hex digits
numSections++
lastSection.deleteRange(0, lastSection.length) // clear last section
}
else -> {
if (zoneIndiceMode) {
if (!isUnreserved(domainArray[index])) {
return false
}
} else {
lastSection.append(domainArray[index]) // collect our possible ipv4 address
if (hexSection && isHex(domainArray[index])) {
hexDigits++
} else {
hexSection = false // non hex digit.
}
}
}
}
if (hexDigits > 4 || numSections > 8) {
return false
}
prevChar = domainArray[index]
index++
}
// numSections != 1 checks for things like: [adf]
// If there are more than 8 sections for the address or there isn't a double colon, then it's invalid.
return numSections != 1 && (numSections >= 8 || doubleColonFlag)
}
companion object {
/**
* The minimum length of a ascii based top level domain.
*/
private const val MIN_TOP_LEVEL_DOMAIN = 2
/**
* The maximum length of a ascii based top level domain.
*/
private const val MAX_TOP_LEVEL_DOMAIN = 22
/**
* The maximum number that the url can be in a url that looks like:
* http://123123123123/path
*/
private const val MAX_NUMERIC_DOMAIN_VALUE = 4294967295L
/**
* The minimum number the url can be in a url that looks like:
* http://123123123123/path
*/
private const val MIN_NUMERIC_DOMAIN_VALUE = 16843008L
/**
* If the domain name is an ip address, for each part of the address, whats the minimum value?
*/
private const val MIN_IP_PART = 0
/**
* If the domain name is an ip address, for each part of the address, whats the maximum value?
*/
private const val MAX_IP_PART = 255
/**
* The start of the utf character code table which indicates that this character is an international character.
* Everything below this value is either a-z,A-Z,0-9 or symbols that are not included in domain name.
*/
private const val INTERNATIONAL_CHAR_START = 192
/**
* The maximum length of each label in the domain name.
*/
private const val MAX_LABEL_LENGTH = 64
/**
* The maximum number of labels in a single domain name.
*/
private const val MAX_NUMBER_LABELS = 127
/**
* The maximum domain name length.
*/
private const val MAX_DOMAIN_LENGTH = 255
}
}
@@ -0,0 +1,101 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector.detection
import com.vitorpamplona.quartz.utils.urldetector.detection.CharUtils.isWhiteSpace
/**
* Class used to read a text input character by character. This also gives the ability to backtrack.
*/
class InputTextReader(
content: String,
) {
/**
* The content to read.
*/
private val content: CharArray = content.toCharArray()
/**
* The current position in the content we are looking at.
*/
var position: Int = 0
private set
/**
* Reads a single char from the content stream and increments the index.
* @return The next available character.
*/
fun read(): Char {
val chr = content[this.position++]
return if (isWhiteSpace(chr)) ' ' else chr
}
/**
* Peeks at the next number of chars and returns as a string without incrementing the current index.
* @param str The string to compare to
*/
fun peekEquals(str: String): Boolean {
if (position + str.length > content.size) return false
for (i in str.indices) {
if (content[position + i] != str[i]) return false
}
return true
}
/**
* Gets the character in the array offset by the current index.
* @param offset The number of characters to offset.
* @return The character at the location of the index plus the provided offset.
*/
fun peekChar(offset: Int): Char {
if (!canReadChars(offset)) {
throw IllegalArgumentException("Index out of bounds")
}
return content[this.position + offset]
}
/**
* Returns true if the reader has more the specified number of chars.
* @param numberChars The number of chars to see if we can read.
* @return True if we can read this number of chars, else false.
*/
fun canReadChars(numberChars: Int): Boolean = content.size >= this.position + numberChars
/**
* Checks if the current stream is at the end.
* @return True if the stream is at the end and no more can be read.
*/
fun eof(): Boolean = content.size <= this.position
/**
* Moves the index to the specified position.
* @param position The position to set the index to.
*/
fun seek(position: Int) {
this.position = position
}
/**
* Goes back a single character.
*/
fun goBack() = this.position--
}
@@ -0,0 +1,597 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector.detection
import com.vitorpamplona.quartz.utils.urldetector.Url
import com.vitorpamplona.quartz.utils.urldetector.UrlMarker
import com.vitorpamplona.quartz.utils.urldetector.UrlPart
import kotlin.math.max
class UrlDetector(
content: String,
) {
/**
* The input stream to read.
*/
private val reader: InputTextReader = InputTextReader(content)
/**
* Buffer to store temporary urls inside of.
*/
private val buffer = StringBuilder()
/**
* Has the scheme been found in this iteration?
*/
private var hasScheme = false
/**
* If the first character in the url is a quote, then look for matching quote at the end.
*/
private var quoteStart = false
/**
* Stores the found urls.
*/
private val urlList: ArrayList<Url> = ArrayList<Url>()
/**
* Keeps track of certain indices to create a Url object.
*/
private var currentUrlMarker: UrlMarker = UrlMarker()
/**
* The states to use to continue writing or not.
*/
enum class ReadEndState {
/**
* The current url is valid.
*/
ValidUrl,
/**
* The current url is invalid.
*/
InvalidUrl,
}
/**
* Detects the urls and returns a list of detected url strings.
* @return A list with detected urls.
*/
fun detect(): List<Url> {
readDefault()
return urlList
}
/**
* The default input reader which looks for specific flags to start detecting the url.
*/
private fun readDefault() {
// Keeps track of the number of characters read to be able to later cut out the domain name.
var length = 0
var position = 0
// until end of string read the contents
while (!reader.eof()) {
// read the next char to process.
when (val curr = reader.read()) {
' ' -> {
buffer.append(curr)
readEnd(ReadEndState.InvalidUrl)
length = 0
}
'%' -> {
if (reader.canReadChars(2)) {
if (reader.peekEquals("3a") || reader.peekEquals("3A")) {
buffer.append(curr)
buffer.append(reader.read())
buffer.append(reader.read())
length = processColon(length)
} else if (CharUtils.isHex(reader.peekChar(0)) && CharUtils.isHex(reader.peekChar(1))) {
buffer.append(curr)
buffer.append(reader.read())
buffer.append(reader.read())
if (!readDomainName(buffer.substring(length))) {
readEnd(ReadEndState.InvalidUrl)
}
length = 0
}
}
}
'\u3002', '\uFF0E', '\uFF61', '.' -> {
buffer.append(curr)
val domain = buffer.substring(length)
if (!readDomainName(domain)) {
readEnd(ReadEndState.InvalidUrl)
}
length = 0
}
'@' -> {
if (buffer.isNotEmpty()) {
currentUrlMarker.setIndex(UrlPart.USERNAME_PASSWORD, length)
buffer.append(curr)
if (!readDomainName(null)) {
readEnd(ReadEndState.InvalidUrl)
}
length = 0
}
}
'[' -> {
val beginning = reader.position
// if it doesn't have a scheme, clear the buffer.
if (!hasScheme) {
buffer.clear()
}
buffer.append(curr)
if (!readDomainName(buffer.substring(length))) {
// if we didn't find an ipv6 address, then check inside the brackets for urls
readEnd(ReadEndState.InvalidUrl)
reader.seek(beginning)
}
length = 0
}
'/' -> {
// "/" was found, then we either read a scheme, or if we already read a scheme, then
// we are reading a url in the format http://123123123/asdf
if (hasScheme) {
// we already have the scheme, so then we already read:
// http://something/ <- if something is all numeric then its a valid url.
// OR we are searching for single level domains. We have buffer length > 1 condition
// to weed out infinite backtrack in cases of html5 roots
// unread this "/" and continue to check the domain name starting from the beginning of the domain
reader.goBack()
if (!readDomainName(buffer.substring(length))) {
readEnd(ReadEndState.InvalidUrl)
}
length = 0
} else {
// we don't have a scheme already, then clear state, then check for html5 root such as: "//google.com/"
// remember the state of the quote when clearing state just in case its "//google.com" so its not cleared.
readEnd(ReadEndState.InvalidUrl)
buffer.append(curr)
hasScheme = readHtml5Root()
length = buffer.length
}
}
':' -> {
// add the ":" to the url and check for scheme/username
buffer.append(curr)
length = processColon(length)
}
else -> {
buffer.append(curr)
}
}
if (position == reader.position) {
// we haven't made any progress, advance by one char
reader.read()
}
position = reader.position
}
}
/**
* We found a ":" and is now trying to read either scheme, username/password
* @param length first index of the previous part (could be beginning of the buffer, beginning of the username/password, or beginning
* @return new index of where the domain starts
*/
private fun processColon(length: Int): Int {
var length = length
if (hasScheme) {
// read it as username/password if it has scheme
if (!readUserPass(length)) {
// unread the ":" so that the domain reader can process it
reader.goBack()
// Check buffer length before clearing it; set length to 0 if buffer is empty
if (buffer.length > 0) {
buffer.deleteRange(buffer.length - 1, buffer.length)
} else {
length = 0
}
val backtrackOnFail: Int = reader.position - buffer.length + length
if (!readDomainName(buffer.substring(length))) {
// go back to length location and restart search
reader.seek(backtrackOnFail)
readEnd(ReadEndState.InvalidUrl)
}
length = 0
} else {
length = 0
}
} else if (readScheme() && buffer.isNotEmpty()) {
hasScheme = true
length = buffer.length // set length to be right after the scheme
} else {
readEnd(ReadEndState.InvalidUrl)
length = 0
}
return length
}
/**
* Checks if the url is in the format:
* //google.com/static/js.js
* @return True if the url is in this format and was matched correctly.
*/
private fun readHtml5Root(): Boolean {
// end of input then go away.
if (reader.eof()) {
return false
}
// read the next character. If its // then return true.
val curr = reader.read()
if (curr == '/') {
buffer.append(curr)
return true
} else {
// if its not //, then go back and reset by 1 character.
reader.goBack()
readEnd(ReadEndState.InvalidUrl)
}
return false
}
/**
* Reads the scheme and allows returns true if the scheme is http(s?):// or ftp(s?)://
* @return True if the scheme was found, else false.
*/
private fun readScheme(): Boolean {
val originalLength: Int = buffer.length
var numSlashes = 0
while (!reader.eof()) {
val curr = reader.read()
// if we match a slash, look for a second one.
if (curr == '/') {
buffer.append(curr)
if (numSlashes == 1) {
// return only if its an approved protocol. This can be expanded to allow others
val schemeStartIndex: Int = findValidSchemeStartIndex(buffer.toString())
if (schemeStartIndex >= 0) {
buffer.deleteRange(0, schemeStartIndex)
currentUrlMarker.setIndex(UrlPart.SCHEME, 0)
return true
} else {
return false
}
}
numSlashes++
} else if (curr == ' ') {
// if we find a space or end of input, then nothing found.
buffer.append(curr)
return false
} else if (curr == '[') { // if we're starting to see an ipv6 address
reader.goBack() // unread the '[', so that we can start looking for ipv6
return false
} else if (originalLength > 0 || numSlashes > 0 || !CharUtils.isAlpha(curr)) {
// if it's not a character a-z or A-Z then assume we aren't matching scheme, but instead
// matching username and password.
reader.goBack()
return readUserPass(0)
}
}
return false
}
private fun findValidSchemeStartIndex(optionalScheme: String): Int {
val optionalSchemeLowercase = optionalScheme.lowercase()
return VALID_SCHEMES
.filter(optionalSchemeLowercase::endsWith)
.map(optionalSchemeLowercase::lastIndexOf)
.firstOrNull() ?: -1
}
/**
* Reads the input and looks for a username and password.
* Handles:
* http://username:password@...
* @param beginningOfUsername Index of the buffer of where the username began
* @return True if a valid username and password was found.
*/
private fun readUserPass(beginningOfUsername: Int): Boolean {
// The start of where we are.
val start: Int = buffer.length
// keep looping until "done"
var done = false
// if we had a dot in the input, then it might be a domain name and not a username and password.
var rollback = false
while (!done && !reader.eof()) {
val curr = reader.read()
// if we hit this, then everything is ok and we are matching a domain name.
if (curr == '@') {
buffer.append(curr)
currentUrlMarker.setIndex(UrlPart.USERNAME_PASSWORD, beginningOfUsername)
return readDomainName("")
} else if (CharUtils.isDot(curr) || curr == '[') {
// everything is still ok, just remember that we found a dot or '[' in case we might need to backtrack
buffer.append(curr)
rollback = true
} else if (curr == '#' || curr == ' ' || curr == '/') {
// one of these characters indicates we are invalid state and should just return.
rollback = true
done = true
} else {
// all else, just append character assuming its ok so far.
buffer.append(curr)
}
}
if (rollback) {
// got to here, so there is no username and password. (We didn't find a @)
val distance: Int = buffer.length - start
buffer.deleteRange(start, buffer.length)
val currIndex: Int = max(reader.position - distance - (if (done) 1 else 0), 0)
reader.seek(currIndex)
return false
} else {
return readEnd(ReadEndState.InvalidUrl)
}
}
/**
* Try to read the current string as a domain name
* @param current The current string used.
* @return Whether the domain is valid or not.
*/
private fun readDomainName(current: String?): Boolean {
val hostIndex: Int =
if (current == null) buffer.length else buffer.length - current.length
currentUrlMarker.setIndex(UrlPart.HOST, hostIndex)
// create the domain name reader and specify the handler that will be called when a quote character
// or something is found.
val reader = DomainNameReader(reader, buffer, current)
// Try to read the dns and act on the response.
val state = reader.readDomainName()
return when (state) {
DomainNameReader.ReaderNextState.ValidDomainName -> {
readEnd(ReadEndState.ValidUrl)
}
DomainNameReader.ReaderNextState.ReadFragment -> {
readFragment()
}
DomainNameReader.ReaderNextState.ReadPath -> {
readPath()
}
DomainNameReader.ReaderNextState.ReadPort -> {
readPort()
}
DomainNameReader.ReaderNextState.ReadQueryString -> {
readQueryString()
}
DomainNameReader.ReaderNextState.ReadUserPass -> {
val host: Int = currentUrlMarker.indexOf(UrlPart.HOST)
currentUrlMarker.unsetIndex(UrlPart.HOST)
readUserPass(host)
}
else -> {
false
}
}
}
/**
* Reads the fragments which is the part of the url starting with #
* @return If a valid fragment was read true, else false.
*/
private fun readFragment(): Boolean {
currentUrlMarker.setIndex(UrlPart.FRAGMENT, buffer.length - 1)
while (!reader.eof()) {
val curr = reader.read()
// if it's the end or space, then a valid url was read.
if (curr == ' ') {
return readEnd(ReadEndState.ValidUrl)
} else {
// otherwise keep appending.
buffer.append(curr)
}
}
// if we are here, anything read is valid.
return readEnd(ReadEndState.ValidUrl)
}
/**
* Try to read the query string.
* @return True if the query string was valid.
*/
private fun readQueryString(): Boolean {
currentUrlMarker.setIndex(UrlPart.QUERY, buffer.length - 1)
while (!reader.eof()) {
val curr = reader.read()
if (curr == '#') { // fragment
buffer.append(curr)
return readFragment()
} else if (curr == ' ') {
// end of query string
return readEnd(ReadEndState.ValidUrl)
} else { // all else add to buffer.
buffer.append(curr)
}
}
// a valid url was read.
return readEnd(ReadEndState.ValidUrl)
}
/**
* Try to read the port of the url.
* @return True if a valid port was read.
*/
private fun readPort(): Boolean {
currentUrlMarker.setIndex(UrlPart.PORT, buffer.length)
// The length of the port read.
var portLen = 0
while (!reader.eof()) {
// read the next one and remember the length
val curr = reader.read()
portLen++
if (curr == '/') {
// continue to read path
buffer.append(curr)
return readPath()
} else if (curr == '?') {
// continue to read query string
buffer.append(curr)
return readQueryString()
} else if (curr == '#') {
// continue to read fragment.
buffer.append(curr)
return readFragment()
} else if (!CharUtils.isNumeric(curr)) {
// if we got here, then what we got so far is a valid url. don't append the current character.
reader.goBack()
// no port found; it was something like google.com:hello.world
if (portLen == 1) {
// remove the ":" from the end.
buffer.deleteRange(buffer.length - 1, buffer.length)
}
currentUrlMarker.unsetIndex(UrlPart.PORT)
return readEnd(ReadEndState.ValidUrl)
} else {
// this is a valid character in the port string.
buffer.append(curr)
}
}
// found a correct url
return readEnd(ReadEndState.ValidUrl)
}
/**
* Tries to read the path
* @return True if the path is valid.
*/
private fun readPath(): Boolean {
currentUrlMarker.setIndex(UrlPart.PATH, buffer.length - 1)
while (!reader.eof()) {
// read the next char
val curr = reader.read()
if (curr == ' ') {
// if end of state and we got here, then the url is valid.
return readEnd(ReadEndState.ValidUrl)
}
// append the char
buffer.append(curr)
// now see if we move to another state.
if (curr == '?') {
// if ? read query string
return readQueryString()
} else if (curr == '#') {
// if # read the fragment
return readFragment()
}
}
// end of input then this url is good.
return readEnd(ReadEndState.ValidUrl)
}
/**
* The url has been read to here. Remember the url if its valid, and reset state.
* @param state The state indicating if this url is valid. If its valid it will be added to the list of urls.
* @return True if the url was valid.
*/
private fun readEnd(state: ReadEndState?): Boolean {
// if the url is valid and greater then 0
if (state == ReadEndState.ValidUrl && buffer.isNotEmpty()) {
// get the last character. if its a quote, cut it off.
val len: Int = buffer.length
if (quoteStart && buffer[len - 1] == '\"') {
buffer.deleteRange(len - 1, len)
}
// Add the url to the list of good urls.
if (buffer.isNotEmpty()) {
currentUrlMarker.originalUrl = buffer.toString()
urlList.add(currentUrlMarker.createUrl())
}
}
// clear out the buffer.
buffer.deleteRange(0, buffer.length)
// reset the state of internal objects.
quoteStart = false
hasScheme = false
currentUrlMarker = UrlMarker()
// return true if valid.
return state == ReadEndState.ValidUrl
}
companion object {
private val VALID_SCHEMES: List<String> =
listOf(
"http://",
"https://",
"ftp://",
"ftps://",
"ws://",
"wss://",
// "nostr:",
// "blossom:",
)
}
}
@@ -20,6 +20,7 @@
*/
package com.vitorpamplona.quartz.nip64Chess
import com.vitorpamplona.quartz.nip31Alts.alt
import com.vitorpamplona.quartz.nip64Chess.game.ChessGameEvent
import kotlin.test.Test
import kotlin.test.assertEquals
@@ -83,7 +84,7 @@ class ChessGameEventTest {
sig = "test_sig",
)
assertEquals(customAltText, testEvent.altText(), "Alt text should be extractable from tags")
assertEquals(customAltText, testEvent.alt(), "Alt text should be extractable from tags")
}
@Test
@@ -98,7 +99,7 @@ class ChessGameEventTest {
sig = "test_sig",
)
assertEquals(null, testEvent.altText(), "Should return null when no alt tag present")
assertEquals(null, testEvent.alt(), "Should return null when no alt tag present")
}
@Test
@@ -0,0 +1,322 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector
import kotlin.test.Test
import kotlin.test.assertEquals
class UrlMarkerTest {
fun testUrlMarker(
testString: String,
scheme: String?,
username: String?,
password: String?,
host: String?,
port: Int,
path: String?,
query: String?,
fragment: String?,
indices: IntArray,
) {
val urlMarker = UrlMarker()
urlMarker.originalUrl = testString
urlMarker.setIndices(indices)
val url = urlMarker.createUrl()
assertEquals(url.host, host, "host, " + testString)
assertEquals(url.path, path, "path, " + testString)
assertEquals(url.scheme, scheme, "scheme, " + testString)
assertEquals(url.username, username, "username, " + testString)
assertEquals(url.password, password, "password, " + testString)
assertEquals(url.port, port, "port, " + testString)
assertEquals(url.query, query, "query, " + testString)
assertEquals(url.fragment, fragment, "fragment, " + testString)
}
@Test
fun test1() =
testUrlMarker(
"hello@hello.com",
"https",
"hello",
"",
"hello.com",
443,
"/",
"",
"",
intArrayOf(-1, 0, 6, -1, -1, -1, -1),
)
@Test
fun test2() =
testUrlMarker(
"http://hello@hello.com",
"http",
"hello",
"",
"hello.com",
80,
"/",
"",
"",
intArrayOf(0, 7, 13, -1, -1, -1, -1),
)
@Test
fun test3() =
testUrlMarker(
"hello@hello.com",
"https",
"hello",
"",
"hello.com",
443,
"/",
"",
"",
intArrayOf(-1, 0, 6, -1, -1, -1, -1),
)
@Test
fun test4() =
testUrlMarker(
"https://user@google.com/h?hello=w#abc",
"https",
"user",
"",
"google.com",
443,
"/h",
"?hello=w",
"#abc",
intArrayOf(0, 8, 13, -1, 23, 25, 33),
)
@Test
fun test5() =
testUrlMarker(
"www.booopp.com:20#fa",
"https",
"",
"",
"www.booopp.com",
20,
"/",
"",
"#fa",
intArrayOf(-1, -1, 0, 15, -1, -1, 17),
)
@Test
fun test6() =
testUrlMarker(
"www.yahooo.com:20?fff#aa",
"https",
"",
"",
"www.yahooo.com",
20,
"/",
"?fff",
"#aa",
intArrayOf(-1, -1, 0, 15, -1, 17, 21),
)
@Test
fun test7() =
testUrlMarker(
"www.google.com#fa",
"https",
"",
"",
"www.google.com",
443,
"/",
"",
"#fa",
intArrayOf(-1, -1, 0, -1, -1, -1, 14),
)
@Test
fun test8() =
testUrlMarker(
"www.google.com?3fd#fa",
"https",
"",
"",
"www.google.com",
443,
"/",
"?3fd",
"#fa",
intArrayOf(-1, -1, 0, -1, -1, 14, 18),
)
@Test
fun test9() =
testUrlMarker(
"//www.google.com/",
"",
"",
"",
"www.google.com",
-1,
"/",
"",
"",
intArrayOf(-1, -1, 2, -1, 16, -1, -1),
)
@Test
fun test10() =
testUrlMarker(
"http://www.google.com/",
"http",
"",
"",
"www.google.com",
80,
"/",
"",
"",
intArrayOf(0, -1, 7, -1, 21, -1, -1),
)
@Test
fun test11() =
testUrlMarker(
"ftp://whosdere:me@google.com/",
"ftp",
"whosdere",
"me",
"google.com",
21,
"/",
"",
"",
intArrayOf(0, 6, 18, -1, 28, -1, -1),
)
@Test
fun test12() =
testUrlMarker(
"ono:doope@fb.net:9090/dhdh",
"https",
"ono",
"doope",
"fb.net",
9090,
"/dhdh",
"",
"",
intArrayOf(-1, 0, 10, 17, 21, -1, -1),
)
@Test
fun test13() =
testUrlMarker(
"ono:a@fboo.com:90/dhdh/@1234",
"https",
"ono",
"a",
"fboo.com",
90,
"/dhdh/@1234",
"",
"",
intArrayOf(-1, 0, 6, 15, 17, -1, -1),
)
@Test
fun test14() =
testUrlMarker(
"fbeoo.net:990/dhdeh/@1234",
"https",
"",
"",
"fbeoo.net",
990,
"/dhdeh/@1234",
"",
"",
intArrayOf(-1, -1, 0, 10, 13, -1, -1),
)
@Test
fun test15() =
testUrlMarker(
"fbeoo:@boop.com/dhdeh/@1234?aj=r",
"https",
"fbeoo",
"",
"boop.com",
443,
"/dhdeh/@1234",
"?aj=r",
"",
intArrayOf(-1, 0, 7, -1, 15, 27, -1),
)
@Test
fun test16() =
testUrlMarker(
"bloop:@noooo.com/doop/@1234",
"https",
"bloop",
"",
"noooo.com",
443,
"/doop/@1234",
"",
"",
intArrayOf(-1, 0, 7, -1, 16, -1, -1),
)
@Test
fun test17() =
testUrlMarker(
"bah.com/lala/@1234/@dfd@df?@dsf#ono",
"https",
"",
"",
"bah.com",
443,
"/lala/@1234/@dfd@df",
"?@dsf",
"#ono",
intArrayOf(-1, -1, 0, -1, 7, 26, 31),
)
@Test
fun test18() =
testUrlMarker(
"https://dewd:dood@www.google.com:20/?why=is&this=test#?@Sdsf",
"https",
"dewd",
"dood",
"www.google.com",
20,
"/",
"?why=is&this=test",
"#?@Sdsf",
intArrayOf(0, 8, 18, 33, 35, 36, 53),
)
}
@@ -0,0 +1,121 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector.detection
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class CharUtilsTest {
@Test
fun testCharUtilsIsHex() {
val arr = charArrayOf('a', 'A', '0', '9')
for (a in arr) {
assertTrue(CharUtils.isHex(a))
}
val arr2 = charArrayOf('~', ';', 'Z', 'g')
for (a in arr2) {
assertFalse(CharUtils.isHex(a))
}
}
@Test
fun testCharUtilsIsNumeric() {
val arr = charArrayOf('0', '4', '6', '9')
for (a in arr) {
assertTrue(CharUtils.isNumeric(a))
}
val arr2 = charArrayOf('a', '~', 'A', 0.toChar())
for (a in arr2) {
assertFalse(CharUtils.isNumeric(a))
}
}
@Test
fun testCharUtilsIsAlpha() {
val arr = charArrayOf('a', 'Z', 'f', 'X')
for (a in arr) {
assertTrue(CharUtils.isAlpha(a))
}
val arr2 = charArrayOf('0', '9', '[', '~')
for (a in arr2) {
assertFalse(CharUtils.isAlpha(a))
}
}
@Test
fun testCharUtilsIsAlphaNumeric() {
val arr = charArrayOf('a', 'G', '3', '9')
for (a in arr) {
assertTrue(CharUtils.isAlphaNumeric(a))
}
val arr2 = charArrayOf('~', '-', '_', '\n')
for (a in arr2) {
assertFalse(CharUtils.isAlphaNumeric(a))
}
}
@Test
fun testCharUtilsIsUnreserved() {
val arr = charArrayOf('-', '.', 'a', '9', 'Z', '_', 'f')
for (a in arr) {
assertTrue(CharUtils.isUnreserved(a))
}
val arr2 = charArrayOf(' ', '!', '(', '\n')
for (a in arr2) {
assertFalse(CharUtils.isUnreserved(a))
}
}
@Test
fun testSplitByDot() {
val stringsToSplit =
listOf(
"192.168.1.1",
"..",
"192%2e168%2e1%2e1",
"asdf",
"192.39%2e1%2E1",
"as\uFF61awe.a3r23.lkajsf0ijr....",
"%2e%2easdf",
"sdoijf%2e",
"ksjdfh.asdfkj.we%2",
"0xc0%2e0x00%2e0x02%2e0xeb",
"",
)
val regex = "[\\.\u3002\uFF0E\uFF61]|%2e|%2E".toRegex()
stringsToSplit.forEach { stringToSplit ->
assertContentEquals(
stringToSplit.split(regex),
CharUtils.splitByDot(stringToSplit),
"Splitting $stringToSplit",
)
}
}
}
@@ -0,0 +1,76 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector.detection
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class InputTextReaderTest {
@Test
fun testSimpleRead() {
val reader = InputTextReader(CONTENT)
for (i in 0..<CONTENT.length) {
assertEquals(reader.read(), CONTENT[i])
}
}
@Test
fun testEOF() {
val reader = InputTextReader(CONTENT)
for (i in 0..<CONTENT.length - 1) {
reader.read()
}
assertFalse(reader.eof())
reader.read()
assertTrue(reader.eof())
}
@Test
fun testGoBack() {
val reader = InputTextReader(CONTENT)
assertEquals(reader.read(), CONTENT[0])
reader.goBack()
assertEquals(reader.read(), CONTENT[0])
assertEquals(reader.read(), CONTENT[1])
assertEquals(reader.read(), CONTENT[2])
reader.goBack()
reader.goBack()
assertEquals(reader.read(), CONTENT[1])
assertEquals(reader.read(), CONTENT[2])
}
@Test
fun testSeek() {
val reader = InputTextReader(CONTENT)
reader.seek(4)
assertEquals(reader.read(), CONTENT[4])
reader.seek(1)
assertEquals(reader.read(), CONTENT[1])
}
companion object {
private val CONTENT = "HELLO WORLD"
}
}
@@ -0,0 +1,643 @@
/*
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.utils.urldetector.detection
import com.vitorpamplona.quartz.utils.urldetector.Url
import kotlin.test.Test
import kotlin.test.assertEquals
class UriDetectionTest {
@Test
fun testBasicString() {
runTest("hello world")
}
@Test
fun testBasicDetect() {
runTest("this is a link: www.google.com", "www.google.com")
}
@Test
fun testSimple() {
runTest(
"http://www.linkedin.com/vshlos",
"http://www.linkedin.com/vshlos",
)
}
@Test
fun testEmailAndNormalUrl() {
runTest(
"my email is vshlosbe@linkedin.com and my site is http://www.linkedin.com/vshlos",
"vshlosbe@linkedin.com",
"http://www.linkedin.com/vshlos",
)
}
@Test
fun testTwoBasicUrls() {
runTest(
"the url google.com is a lot better then www.google.com.",
"google.com",
"www.google.com.",
)
}
@Test
fun testLongUrl() {
runTest(
"google.com.google.com is kind of a valid url",
"google.com.google.com",
)
}
@Test
fun testInternationalUrls() {
runTest(
"this is an international domain: http://\u043F\u0440\u0438\u043c\u0435\u0440.\u0438\u0441\u043f\u044b" +
"\u0442\u0430\u043d\u0438\u0435 so is this: \u4e94\u7926\u767c\u5c55.\u4e2d\u570b.",
"http://\u043F\u0440\u0438\u043c\u0435\u0440.\u0438\u0441\u043f\u044b\u0442\u0430\u043d\u0438\u0435",
"\u4e94\u7926\u767c\u5c55.\u4e2d\u570b.",
)
}
@Test
fun testDomainWithUsernameAndPassword() {
runTest(
"domain with username is http://username:password@www.google.com/site/1/2",
"http://username:password@www.google.com/site/1/2",
)
}
@Test
fun testFTPWithUsernameAndPassword() {
runTest(
"ftp with username is ftp://username:password@www.google.com",
"ftp://username:password@www.google.com",
)
}
@Test
fun testUncommonFormatUsernameAndPassword() {
runTest(
"weird url with username is username:password@www.google.com",
"username:password@www.google.com",
)
}
@Test
fun testEmailAndLinkWithUserPass() {
runTest(
"email and username is hello@test.google.com or hello@www.google.com hello:password@www.google.com",
"hello@test.google.com",
"hello@www.google.com",
"hello:password@www.google.com",
)
}
@Test
fun testWrongSpacingInSentence() {
runTest(
"I would not like to work at salesforce.com, it looks like a crap company.and not cool!",
"salesforce.com",
"company.and",
)
}
@Test
fun testNumbersAreNotDetected() {
// make sure pure numbers don't work, but domains with numbers do.
runTest("Do numbers work? such as 3.1415 or 4.com", "4.com")
}
@Test
fun testNewLinesAndTabsAreDelimiters() {
runTest(
"Do newlines and tabs break? google.com/hello/\nworld www.yahoo.com\t/stuff/ yahoo.com/\thello news.ycombinator.com\u0000/hello world",
"google.com/hello/",
"www.yahoo.com",
"yahoo.com/",
"news.ycombinator.com",
)
}
@Test
fun testIpAddressFormat() {
runTest(
"How about IP addresses? fake: 1.1.1 1.1.1.1.1 0.0.0.256 255.255.255.256 real: 1.1.1.1 192.168.10.1 1.1.1.1.com 255.255.255.255",
"1.1.1.1",
"192.168.10.1",
"1.1.1.1.com",
"255.255.255.255",
)
}
@Test
fun testNumericIpAddress() {
runTest(
"http://3232235521/helloworld",
"http://3232235521/helloworld",
)
}
@Test
fun testNumericIpAddressWithPort() {
runTest(
"http://3232235521:8080/helloworld",
"http://3232235521:8080/helloworld",
)
}
@Test
fun testDomainAndLabelSizeConstraints() {
// Really long addresses testing rules about total length of domain name and number of labels in a domain and size of each label.
runTest(
(
"This will work: 1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.a.b.c.d.e.ly " +
"This will not work: 1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.a.b.c.d.e.f.ly " +
"This should as well: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc.dddddddddddddddddddddddddddddddddddddddddddddddddddddd.bit.ly " +
"But this wont: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc.dddddddddddddddddddddddddddddddddddddddddddddddddddddd.bit.ly.dbl.spamhaus.org"
),
"1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.1.2.3.4.5.6.7.8.9.0.a.b.c.d.e.ly",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc.dddddddddddddddddddddddddddddddddddddddddddddddddddddd.bit.ly",
)
}
@Test
fun testIncorrectParsingHtmlWithBadOptions() {
runTest(
"<a href=\"http://www.google.com/\">google.com</a>",
"http://www.google.com/\">google.com</a>",
)
}
@Test
fun testNonStandardDots() {
runTest(
"www\u3002google\u3002com username:password@www\uFF0Eyahoo\uFF0Ecom http://www\uFF61facebook\uFF61com http://192\u3002168\uFF0E0\uFF611/",
"www\u3002google\u3002com",
"username:password@www\uFF0Eyahoo\uFF0Ecom",
"http://www\uFF61facebook\uFF61com",
"http://192\u3002168\uFF0E0\uFF611/",
)
}
@Test
fun testNonStandardDotsBacktracking() {
runTest("\u9053 \u83dc\u3002\u3002\u3002\u3002")
}
@Test
fun testBacktrackingStrangeFormats() {
runTest(
"http:http:http://www.google.com www.www:yahoo.com yahoo.com.br hello.hello..hello.com",
"http://www.google.com",
"www.www",
"yahoo.com",
"yahoo.com.br",
"hello.hello.",
"hello.com",
)
}
@Test
fun testBacktrackingUsernamePassword() {
runTest("check out my url:www.google.com", "www.google.com")
runTest("check out my url:www.google.com ", "www.google.com")
}
@Test
fun testBacktrackingEmptyDomainName() {
runTest("check out my http:///hello")
runTest("check out my http://./hello")
}
@Test
fun testDoubleScheme() {
runTest("http://http://")
runTest("hello http://http://")
}
@Test
fun testMultipleSchemes() {
runTest("http://http://www.google.com", "http://www.google.com")
runTest(
"make sure it's right here http://http://www.google.com",
"http://www.google.com",
)
runTest(
"http://http://http://www.google.com",
"http://www.google.com",
)
runTest(
"make sure it's right here http://http://http://www.google.com",
"http://www.google.com",
)
runTest(
"http://ftp://https://www.google.com",
"https://www.google.com",
)
runTest(
"make sure its right here http://ftp://https://www.google.com",
"https://www.google.com",
)
}
@Test
fun testDottedHexIpAddress() {
runTest(
"http://0xc0.0x00.0xb2.0xEB",
"http://0xc0.0x00.0xb2.0xEB",
)
runTest(
"http://0xc0.0x0.0xb2.0xEB",
"http://0xc0.0x0.0xb2.0xEB",
)
runTest(
"http://0x000c0.0x00000.0xb2.0xEB",
"http://0x000c0.0x00000.0xb2.0xEB",
)
runTest(
"http://0xc0.0x00.0xb2.0xEB/bobo",
"http://0xc0.0x00.0xb2.0xEB/bobo",
)
runTest(
"ooh look i can find it in text http://0xc0.0x00.0xb2.0xEB/bobo like this",
"http://0xc0.0x00.0xb2.0xEB/bobo",
)
runTest(
"noscheme look 0xc0.0x00.0xb2.0xEB/bobo",
"0xc0.0x00.0xb2.0xEB/bobo",
)
runTest(
"no scheme 0xc0.0x00.0xb2.0xEB or path",
"0xc0.0x00.0xb2.0xEB",
)
}
@Test
fun testDottedOctalIpAddress() {
runTest(
"http://0301.0250.0002.0353",
"http://0301.0250.0002.0353",
)
runTest(
"http://0301.0250.0002.0353/bobo",
"http://0301.0250.0002.0353/bobo",
)
runTest("http://192.168.017.015/", "http://192.168.017.015/")
runTest(
"ooh look i can find it in text http://0301.0250.0002.0353/bobo like this",
"http://0301.0250.0002.0353/bobo",
)
runTest(
"noscheme look 0301.0250.0002.0353/bobo",
"0301.0250.0002.0353/bobo",
)
runTest(
"no scheme 0301.0250.0002.0353 or path",
"0301.0250.0002.0353",
)
}
@Test
fun testHexIpAddress() {
runTest("http://0xC00002EB/hello", "http://0xC00002EB/hello")
runTest(
"http://0xC00002EB.com/hello",
"http://0xC00002EB.com/hello",
)
runTest(
"still look it up as a normal url http://0xC00002EXsB.com/hello",
"http://0xC00002EXsB.com/hello",
)
runTest(
"ooh look i can find it in text http://0xC00002EB/bobo like this",
"http://0xC00002EB/bobo",
)
runTest(
"browsers dont support this without a scheme look 0xC00002EB/bobo",
)
}
@Test
fun testOctalIpAddress() {
runTest(
"http://030000001353/bobobo",
"http://030000001353/bobobo",
)
runTest(
"ooh look i can find it in text http://030000001353/bobo like this",
"http://030000001353/bobo",
)
runTest(
"browsers dont support this without a scheme look 030000001353/bobo",
)
}
@Test
fun testUrlWithEmptyPort() {
runTest(
"http://wtfismyip.com://foo.html",
"http://wtfismyip.com://foo.html",
)
runTest(
"make sure its right here http://wtfismyip.com://foo.html",
"http://wtfismyip.com://foo.html",
)
}
@Test
fun testUrlEncodedDot() {
runTest("hello www%2ewtfismyip%2ecom", "www%2ewtfismyip%2ecom")
runTest("hello wtfismyip%2ecom", "wtfismyip%2ecom")
runTest("http://wtfismyip%2ecom", "http://wtfismyip%2ecom")
runTest(
"make sure its right here http://wtfismyip%2ecom",
"http://wtfismyip%2ecom",
)
}
@Test
fun testUrlEncodedBadPath() {
runTest("%2ewtfismyip")
runTest("wtfismyip%2e")
runTest("wtfismyip%2ecom%2e", "wtfismyip%2ecom%2e")
runTest("wtfismyip%2ecom.", "wtfismyip%2ecom.")
runTest("%2ewtfismyip%2ecom", "wtfismyip%2ecom")
}
@Test
fun testDetectUrlEncoded() {
runTest(
"%77%77%77%2e%67%75%6d%62%6c%61%72%2e%63%6e",
"%77%77%77%2e%67%75%6d%62%6c%61%72%2e%63%6e",
)
runTest(
" asdf %77%77%77%2e%67%75%6d%62%6c%61%72%2e%63%6e",
"%77%77%77%2e%67%75%6d%62%6c%61%72%2e%63%6e",
)
runTest(
"%77%77%77%2e%67%75%6d%62%6c%61%72%2e%63%6e%2e",
"%77%77%77%2e%67%75%6d%62%6c%61%72%2e%63%6e%2e",
)
}
@Test
fun testIncompleteIpAddresses() {
runTest("hello 10...")
runTest("hello 10...1")
runTest("hello 10..1.")
runTest("hello 10..1.1")
runTest("hello 10.1..1")
runTest("hello 10.1.1.")
runTest("hello .192..")
runTest("hello .192..1")
runTest("hello .192.1.")
runTest("hello .192.1.1")
runTest("hello ..3.")
runTest("hello ..3.1")
runTest("hello ...1")
}
@Test
fun testIPv4EncodedDot() {
runTest("hello 192%2e168%2e1%2e1", "192%2e168%2e1%2e1")
runTest(
"hello 192.168%2e1%2e1/lalala",
"192.168%2e1%2e1/lalala",
)
}
@Test
fun testIPv4HexEncodedDot() {
runTest(
"hello 0xee%2e0xbb%2e0x1%2e0x1",
"0xee%2e0xbb%2e0x1%2e0x1",
)
runTest(
"hello 0xee%2e0xbb.0x1%2e0x1/lalala",
"0xee%2e0xbb.0x1%2e0x1/lalala",
)
}
@Test
fun testIpv6BadWithGoodUrls() {
runTest("[:::] [::] [bacd::]", "[::]", "[bacd::]")
runTest("[:0][::]", "[::]")
runTest("[:0:][::afaf]", "[::afaf]")
runTest(
"::] [fe80:aaaa:aaaa:aaaa::]",
"[fe80:aaaa:aaaa:aaaa::]",
)
runTest(
"fe80:22:]3123:[adf] [fe80:aaaa:aaaa:aaaa::]",
"[fe80:aaaa:aaaa:aaaa::]",
)
runTest("[][123[][ae][fae][de][:a][d]aef:E][f")
runTest("[][][]2[d][]][]]]:d][[[:d[e][aee:]af:")
}
@Test
fun testIpv6BadWithGoodUrlsEmbedded() {
runTest(
"[fe80:aaaa:aaaa:aaaa:[::]3dd0:7f8e:57b7:34d5f]",
"[::]",
)
runTest("[b[::7f8e]:55]akjef[::]", "[::7f8e]:55", "[::]")
runTest(
"[bcad::kkkk:aaaa:3dd0[::7f8e]:57b7:34d5]akjef[::]",
"[::7f8e]:57",
"[::]",
)
}
@Test
fun testIpv6BadWithGoodUrlsWeirder() {
runTest("[:[::]", "[::]")
runTest("[:] [feed::]", "[feed::]")
runTest(":[::feee]:]", "[::feee]")
runTest(":[::feee]:]]", "[::feee]")
runTest("[[:[::feee]:]", "[::feee]")
}
@Test
fun testIpv6ConsecutiveGoodUrls() {
runTest("[::afaf][eaea::][::]", "[::afaf]", "[eaea::]", "[::]")
runTest("[::afaf]www.google.com", "[::afaf]", "www.google.com")
runTest("[lalala:we][::]", "[::]")
runTest("[::fe][::]", "[::fe]", "[::]")
runTest("[aaaa::][:0:][::afaf]", "[aaaa::]", "[::afaf]")
}
@Test
fun testIpv6BacktrackingUsernamePassword() {
runTest("check out my url:google.com", "google.com")
runTest(
"check out my url:[::BAD:DEAD:BEEF:2e80:0:0]",
"[::BAD:DEAD:BEEF:2e80:0:0]",
)
runTest(
"check out my url:[::BAD:DEAD:BEEF:2e80:0:0] ",
"[::BAD:DEAD:BEEF:2e80:0:0]",
)
}
@Test
fun testIpv6BacktrackingEmptyDomainName() {
runTest("check out my http:///[::2e80:0:0]", "[::2e80:0:0]")
runTest("check out my http://./[::2e80:0:0]", "[::2e80:0:0]")
}
@Test
fun testIpv6DoubleSchemeWithDomain() {
runTest("http://http://[::2e80:0:0]", "http://[::2e80:0:0]")
runTest(
"make sure its right here http://http://[::2e80:0:0]",
"http://[::2e80:0:0]",
)
}
@Test
fun testIpv6MultipleSchemes() {
runTest(
"http://http://http://[::2e80:0:0]",
"http://[::2e80:0:0]",
)
runTest(
"make sure its right here http://http://[::2e80:0:0]",
"http://[::2e80:0:0]",
)
runTest(
"http://ftp://https://[::2e80:0:0]",
"https://[::2e80:0:0]",
)
runTest(
"make sure its right here http://ftp://https://[::2e80:0:0]",
"https://[::2e80:0:0]",
)
}
@Test
fun testIpv6FtpWithUsernameAndPassword() {
runTest(
"ftp with username is ftp://username:password@[::2e80:0:0]",
"ftp://username:password@[::2e80:0:0]",
)
}
@Test
fun testIpv6NewLinesAndTabsAreDelimiters() {
runTest(
"Do newlines and tabs break? [::2e80:0:0]/hello/\nworld [::BEEF:ADD:BEEF]\t/stuff/ [AAbb:AAbb:AAbb::]/\thello [::2e80:0:0\u0000]/hello world",
"[::2e80:0:0]/hello/",
"[::BEEF:ADD:BEEF]",
"[AAbb:AAbb:AAbb::]/",
)
}
@Test
fun testIpv6WithPort() {
runTest(
"http://[AAbb:AAbb:AAbb::]:8080/helloworld",
"http://[AAbb:AAbb:AAbb::]:8080/helloworld",
)
}
@Test
fun testIpv6IncorrectParsingHtmlWithBadOptions() {
runTest(
"<a href=\"http://[::AAbb:]/\">google.com</a>",
"http://[::AAbb:]/\">google.com</a>",
)
}
@Test
fun testIpv6EmptyPort() {
runTest(
"http://[::AAbb:]://foo.html",
"http://[::AAbb:]://foo.html",
)
runTest(
"make sure its right here http://[::AAbb:]://foo.html",
"http://[::AAbb:]://foo.html",
)
}
@Test
fun testBacktrackInvalidUsernamePassword() {
runTest("http://hello:asdf.com", "asdf.com")
}
/*
* https://github.com/linkedin/URL-Detector/issues/12
*/
@Test
fun testIssue12() {
runTest(
"http://user:pass@host.com host.com",
"http://user:pass@host.com",
"host.com",
)
}
/*
* https://github.com/linkedin/URL-Detector/issues/15
*/
@Test
fun testIssue15() {
runTest(
".............:::::::::::;;;;;;;;;;;;;;;::...............................................:::::::::::::::::::::::::::::....................",
)
}
/*
* https://github.com/linkedin/URL-Detector/issues/16
*/
@Test
fun testIssue16() {
runTest("://VIVE MARINE LE PEN//:@.")
}
@Test
fun testColonWithoutSlashesFail() {
val parser = UrlDetector("ftp:example.com")
val found: List<Url> = parser.detect()
for (url in found) {
assertEquals(url.scheme, "https")
// Should be detected as a username now and set to default http://
assertEquals(url.host, "example.com")
}
}
@Test
fun testIssueUnderscore() {
runTest("Neomobius_at_mstdn.jp@mostr.pub", "Neomobius_at_mstdn.jp@mostr.pub")
}
private fun runTest(
text: String,
vararg expected: String?,
) = assertEquals(
expected.toList(),
UrlDetector(text).detect().map { it.originalUrl },
)
}
@@ -20,7 +20,6 @@
*/
package com.vitorpamplona.quartz.utils
import com.linkedin.urls.detection.UrlDetector
import com.linkedin.urls.detection.UrlDetectorOptions
import com.vitorpamplona.quartz.utils.urldetector.detection.UrlDetector
actual fun fastFindURLs(text: String): List<String> = UrlDetector(text, UrlDetectorOptions.Default).detect().map { it.originalUrl }
actual fun fastFindURLs(text: String): List<String> = UrlDetector(text).detect().map { it.originalUrl }
@@ -20,6 +20,9 @@
*/
package com.vitorpamplona.quartz.nip64Chess
import com.vitorpamplona.quartz.nip64Chess.jester.JesterEvent
import com.vitorpamplona.quartz.nip64Chess.jester.JesterGameEvents
import com.vitorpamplona.quartz.nip64Chess.jester.JesterProtocol
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse