From fd28d354e7458c7e07cc8fd880a7d834ddea1ccb Mon Sep 17 00:00:00 2001 From: kojira Date: Sun, 1 Mar 2026 23:24:19 +0900 Subject: [PATCH] fix: treat multibyte characters as URL terminators in RichTextParser --- .../commons/richtext/RichTextParser.kt | 19 ++- .../richtext/RichTextParserMultibyteTest.kt | 132 ++++++++++++++++++ 2 files changed, 140 insertions(+), 11 deletions(-) create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMultibyteTest.kt diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt index 615d64ce8..8cde06a74 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt @@ -175,7 +175,7 @@ class RichTextParser { lines.forEach { paragraph -> val isRTL = isArabic(paragraph) - val wordList = paragraph.trimEnd().split(' ') + val wordList = paragraph.trimEnd().split(wordBoundaryRegex).filter { it.isNotEmpty() } val segments = ArrayList(wordList.size) wordList.forEach { word -> segments.add(wordIdentifier(word, images, videos, urls, emojis, tags)) @@ -334,17 +334,14 @@ class RichTextParser { val shortDatePattern: Regex = Regex("^\\d{2}-\\d{2}-\\d{2}$") val numberPattern: Regex = Regex("^(-?[\\d.]+)([a-zA-Z%]*)$") - // Android9 seems to have an issue starting this regex. val noProtocolUrlValidator = - try { - Regex( - "(([\\w\\d-]+\\.)*[a-zA-Z][\\w-]+[\\.\\:]\\w+([\\/\\?\\=\\&\\#\\.]?[\\w-]+[^\\p{IsHan}\\p{IsHiragana}\\p{IsKatakana}])*\\/?)(.*)", - ) - } catch (e: Exception) { - Regex( - "(([\\w\\d-]+\\.)*[a-zA-Z][\\w-]+[\\.\\:]\\w+([\\/\\?\\=\\&\\#\\.]?[\\w-]+)*\\/?)(.*)", - ) - } + Regex( + "(([a-zA-Z0-9_-]+\\.)*[a-zA-Z][a-zA-Z0-9_-]+[\\.\\:][a-zA-Z0-9_]+([\\/ \\?\\=\\&\\#\\.]?[a-zA-Z0-9_-]+)*\\/?)(.*)", + ) + + // Splits at spaces AND at ASCII/multibyte character boundaries + // e.g. "ああexample.com" -> ["ああ", "example.com"] + private val wordBoundaryRegex = Regex("(?<=[\\u0000-\\u007F])(?=[\\u0080-\\uFFFF])|(?<=[\\u0080-\\uFFFF])(?=[\\u0000-\\u007F])| +") val additionalUrlSchema = """^([A-Za-z0-9-_]+(\.[A-Za-z0-9-_]+)+)(:[0-9]+)?(/[^?#]*)?(\?[^#]*)?(#.*)?""" diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMultibyteTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMultibyteTest.kt new file mode 100644 index 000000000..75af7b8a0 --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMultibyteTest.kt @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.richtext + +import com.vitorpamplona.amethyst.commons.model.EmptyTagList +import org.junit.Assert.assertTrue +import org.junit.Test + +class RichTextParserMultibyteTest { + @Test + fun testFullTextWithMultibyte() { + // Multibyte characters around an email address should not produce URL/Link segments + val text = + "マルチバイト文字テストuser@example.com ほげほげ" + + val state = + RichTextParser() + .parseText(text, EmptyTagList, null) + + val allSegments = + state.paragraphs + .flatMap { it.words } + + // user@example.com should be EmailSegment + assertTrue( + "user@example.com should be EmailSegment", + allSegments.any { it is EmailSegment && it.segmentText == "user@example.com" }, + ) + + // user@example.com should NOT be a LinkSegment + assertTrue( + "user@example.com should not be a LinkSegment", + allSegments.none { it is LinkSegment && it.segmentText == "user@example.com" }, + ) + + // user@example.com should not be in urlSet + assertTrue( + "user@example.com should not be in urlSet", + !state.urlSet.contains("user@example.com"), + ) + } + + @Test + fun testEmailSegmentStandalone() { + val text = "user@example.com" + val state = RichTextParser().parseText(text, EmptyTagList, null) + val allSegments = state.paragraphs.flatMap { it.words } + assertTrue( + "user@example.com should be EmailSegment", + allSegments.any { it is EmailSegment && it.segmentText == "user@example.com" }, + ) + } + + @Test + fun testMultibytePrefix_SchemelessUrl() { + // ああexample.com → RegularText(ああ) + SchemelessUrl(example.com) + val text = "ああexample.com" + val state = RichTextParser().parseText(text, EmptyTagList, null) + val allSegments = state.paragraphs.flatMap { it.words } + + val urlSegments = allSegments.filterIsInstance() + assertTrue("Should have SchemelessUrlSegment", urlSegments.isNotEmpty()) + assertTrue("URL should be example.com", urlSegments.any { it.url == "example.com" }) + + val textSegments = allSegments.filterIsInstance() + assertTrue("Should have prefix ああ", textSegments.any { it.segmentText == "ああ" }) + } + + @Test + fun testMultibyteSuffix_SchemelessUrl() { + // example.comああ → SchemelessUrl(example.com) + RegularText(ああ) + val text = "example.comああ" + val state = RichTextParser().parseText(text, EmptyTagList, null) + val allSegments = state.paragraphs.flatMap { it.words } + + val urlSegments = allSegments.filterIsInstance() + assertTrue("Should have SchemelessUrlSegment", urlSegments.isNotEmpty()) + assertTrue("URL should be example.com", urlSegments.any { it.url == "example.com" }) + + val textSegments = allSegments.filterIsInstance() + assertTrue("Should have suffix ああ", textSegments.any { it.segmentText == "ああ" }) + } + + @Test + fun testMultibytePrefix_Email() { + // ほむほむuser@example.comほげほげ → RegularText(ほむほむ) + Email(user@example.com) + RegularText(ほげほげ) + val text = "ほむほむuser@example.comほげほげ" + val state = RichTextParser().parseText(text, EmptyTagList, null) + val allSegments = state.paragraphs.flatMap { it.words } + + val emailSegment = allSegments.filterIsInstance() + assertTrue("Should have EmailSegment", emailSegment.isNotEmpty()) + assertTrue("Email should be user@example.com", emailSegment.any { it.segmentText == "user@example.com" }) + + val textSegments = allSegments.filterIsInstance() + assertTrue("Should have prefix ほむほむ", textSegments.any { it.segmentText == "ほむほむ" }) + assertTrue("Should have suffix ほげほげ", textSegments.any { it.segmentText == "ほげほげ" }) + } + + @Test + fun testEmailWithSpaceAndMultibyteText() { + // user@example.com ふがふが → Email(user@example.com) + RegularText(ふがふが) + val text = "user@example.com ふがふが" + val state = RichTextParser().parseText(text, EmptyTagList, null) + val allSegments = state.paragraphs.flatMap { it.words } + + val emailSegment = allSegments.filterIsInstance() + assertTrue("Should have EmailSegment", emailSegment.isNotEmpty()) + assertTrue("Email should be user@example.com", emailSegment.any { it.segmentText == "user@example.com" }) + + val textSegments = allSegments.filterIsInstance() + assertTrue("Should have suffix ふがふが", textSegments.any { it.segmentText == "ふがふが" }) + } +}