test(translation): JVM unit tests for placeholder dictionary round-trip
Extracts the placeholder dictionary logic out of LanguageTranslatorService into a
pure-JVM TranslationDictionary helper so the round-trip can be unit-tested
without ML Kit / Android runtime, and adds 24 tests covering it.
The existing TranslationsTest is androidTestPlay-only — it needs a real device or
emulator with Google Play services, so it can't validate a refactor in plain CI
or local dev. The new tests cover the riskiest part of this branch: that PUA
placeholders survive an arbitrary "translation" of the surrounding text and
decode back to the exact original tokens.
Test coverage
- isWorthTranslating: short text / letterless text rejected, mixed letters+emoji accepted
- placeholder: produces single PUA codepoint, rejects out-of-range index
- build: collects URLs, NIP-19 nostr refs, Lightning invoices, NIP-08 #[N] refs
- build: deduplicates repeated occurrences and skips Chinese-punctuation URL false-positives
- encode/decode round-trip on plain URLs, multi-URL strings, and mixed real-world content
- encode replaces longer values first to avoid prefix collisions
- decode preserves user text containing the OLD "B0/C0/A0" tokens (regression for the
pre-rewrite collision bug)
- case-sensitive replacement preserves user text that differs only in case from a placeholder
- decode handles null and empty-dictionary inputs
- simulated translation (rewrite English to Portuguese around the placeholders) round-trips
#[0] and nostr:nevent1... unchanged
LanguageTranslatorService now delegates to TranslationDictionary.{build, encode,
decode, isWorthTranslating} — public API (autoTranslate / translate /
identifyLanguage / clear) and behaviour are unchanged.
Result: 410 tests run, 408 passed, 2 pre-existing skips, 0 failures.
https://claude.ai/code/session_0153e2sVbAijKxinQYa6cNx5
This commit is contained in:
+4
-97
@@ -31,11 +31,9 @@ import com.google.mlkit.nl.translate.Translation
|
||||
import com.google.mlkit.nl.translate.Translator
|
||||
import com.google.mlkit.nl.translate.TranslatorOptions
|
||||
import com.vitorpamplona.amethyst.service.checkNotInMainThread
|
||||
import com.vitorpamplona.quartz.utils.urldetector.detection.UrlDetector
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@Immutable
|
||||
data class ResultOrError(
|
||||
@@ -45,17 +43,6 @@ data class ResultOrError(
|
||||
)
|
||||
|
||||
object LanguageTranslatorService {
|
||||
// Texts shorter than this, or with no letters at all (emoji-only, punctuation), are skipped
|
||||
// before any ML Kit work — language identification is unreliable on them anyway.
|
||||
private const val MIN_TRANSLATABLE_LENGTH = 4
|
||||
|
||||
// Single Unicode Private Use Area codepoint per placeholder. PUA chars don't appear in normal
|
||||
// user text, the translator has no rule for them so it passes them through, and using one
|
||||
// codepoint (instead of bracketed digits) means the translator can't split or reorder the
|
||||
// placeholder. Range U+E000..U+F8FF gives 6400 slots, far more than any single note needs.
|
||||
private const val PLACEHOLDER_BASE = 0xE000
|
||||
private const val PLACEHOLDER_LIMIT = 0xF8FF - PLACEHOLDER_BASE
|
||||
|
||||
private val executorService: ExecutorService =
|
||||
Executors.newFixedThreadPool(maxOf(2, Runtime.getRuntime().availableProcessors() / 2))
|
||||
|
||||
@@ -67,17 +54,6 @@ object LanguageTranslatorService {
|
||||
.build()
|
||||
private val languageIdentification = LanguageIdentification.getClient(identificationOptions)
|
||||
|
||||
val lnRegex: Pattern = Pattern.compile("\\blnbc[a-z0-9]+\\b", Pattern.CASE_INSENSITIVE)
|
||||
val tagRegex: Pattern =
|
||||
Pattern.compile(
|
||||
"(nostr:)?@?(nsec1|npub1|nevent1|naddr1|note1|nprofile1|nrelay1)([qpzry9x8gf2tvdw0s3jn54khce6mua7l]+)",
|
||||
Pattern.CASE_INSENSITIVE,
|
||||
)
|
||||
|
||||
// Legacy NIP-08 positional references like #[0]. Translators tend to insert a space inside the
|
||||
// brackets ("# [0]"), so we shield them via the placeholder dictionary instead of post-fixing.
|
||||
val nip08RefRegex: Pattern = Pattern.compile("#\\[\\d+]")
|
||||
|
||||
private val translators =
|
||||
object : LruCache<TranslatorOptions, Translator>(3) {
|
||||
override fun create(options: TranslatorOptions): Translator = Translation.getClient(options)
|
||||
@@ -136,12 +112,12 @@ object LanguageTranslatorService {
|
||||
return translator.downloadModelIfNeeded().onSuccessTask(executorService) {
|
||||
checkNotInMainThread()
|
||||
|
||||
val dict = buildDictionary(text)
|
||||
val encoded = encodeWithDictionary(text, dict)
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
|
||||
translator.translate(encoded).continueWith(executorService) { task ->
|
||||
task.exception?.let { throw it }
|
||||
ResultOrError(decodeWithDictionary(task.result, dict), source, target)
|
||||
ResultOrError(TranslationDictionary.decode(task.result, dict), source, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -151,7 +127,7 @@ object LanguageTranslatorService {
|
||||
dontTranslateFrom: Set<String>,
|
||||
translateTo: String,
|
||||
): Task<ResultOrError> {
|
||||
if (!isWorthTranslating(text)) return Tasks.forCanceled()
|
||||
if (!TranslationDictionary.isWorthTranslating(text)) return Tasks.forCanceled()
|
||||
|
||||
val key = InFlightKey(text, translateTo, dontTranslateFrom)
|
||||
inFlight[key]?.let { return it }
|
||||
@@ -171,73 +147,4 @@ object LanguageTranslatorService {
|
||||
winner.addOnCompleteListener(executorService) { inFlight.remove(key, winner) }
|
||||
return winner
|
||||
}
|
||||
|
||||
private fun isWorthTranslating(text: String): Boolean {
|
||||
if (text.length < MIN_TRANSLATABLE_LENGTH) return false
|
||||
// Cheap scan; bail as soon as we see one letter codepoint.
|
||||
for (cp in text.codePoints()) {
|
||||
if (Character.isLetter(cp)) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private fun buildDictionary(text: String): Map<String, String> {
|
||||
val dict = LinkedHashMap<String, String>()
|
||||
var counter = 0
|
||||
|
||||
fun addUnique(value: String) {
|
||||
if (value.isEmpty()) return
|
||||
if (counter > PLACEHOLDER_LIMIT) return
|
||||
if (dict.containsValue(value)) return
|
||||
dict[placeholder(counter++)] = value
|
||||
}
|
||||
|
||||
val lnMatcher = lnRegex.matcher(text)
|
||||
while (lnMatcher.find()) addUnique(lnMatcher.group())
|
||||
|
||||
val tagMatcher = tagRegex.matcher(text)
|
||||
while (tagMatcher.find()) addUnique(tagMatcher.group())
|
||||
|
||||
val nip08Matcher = nip08RefRegex.matcher(text)
|
||||
while (nip08Matcher.find()) addUnique(nip08Matcher.group())
|
||||
|
||||
for (url in UrlDetector(text).detect()) {
|
||||
val original = url.originalUrl
|
||||
// The URL detector greedily includes Chinese full-width punctuation; skip those false hits.
|
||||
if (original.contains(',') || original.contains('。')) continue
|
||||
addUnique(original)
|
||||
}
|
||||
|
||||
return dict
|
||||
}
|
||||
|
||||
private fun placeholder(index: Int): String {
|
||||
require(index in 0..PLACEHOLDER_LIMIT) { "placeholder index $index out of range" }
|
||||
return String(Character.toChars(PLACEHOLDER_BASE + index))
|
||||
}
|
||||
|
||||
private fun encodeWithDictionary(
|
||||
text: String,
|
||||
dict: Map<String, String>,
|
||||
): String {
|
||||
if (dict.isEmpty()) return text
|
||||
var newText = text
|
||||
// Replace longest values first so a URL prefix never clobbers a longer URL or tag.
|
||||
for ((token, original) in dict.entries.sortedByDescending { it.value.length }) {
|
||||
newText = newText.replace(original, token, ignoreCase = false)
|
||||
}
|
||||
return newText
|
||||
}
|
||||
|
||||
private fun decodeWithDictionary(
|
||||
text: String?,
|
||||
dict: Map<String, String>,
|
||||
): String? {
|
||||
if (text == null || dict.isEmpty()) return text
|
||||
var newText: String = text
|
||||
for ((token, original) in dict) {
|
||||
newText = newText.replace(token, original, ignoreCase = false)
|
||||
}
|
||||
return newText
|
||||
}
|
||||
}
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.amethyst.service.lang
|
||||
|
||||
import com.vitorpamplona.quartz.utils.urldetector.detection.UrlDetector
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* Pure-JVM helpers that protect non-translatable substrings (URLs, Lightning invoices, NIP-19
|
||||
* references, NIP-08 positional references) by swapping them with single Unicode Private Use Area
|
||||
* codepoints around a translator. Extracted out of [LanguageTranslatorService] so the round-trip
|
||||
* can be unit-tested without ML Kit / Android runtime.
|
||||
*/
|
||||
internal object TranslationDictionary {
|
||||
// Range U+E000..U+F8FF gives 6400 placeholder slots. PUA codepoints don't appear in normal
|
||||
// user text, the translator has no rule for them so it passes them through, and using one
|
||||
// codepoint per placeholder means the translator can't split or reorder it.
|
||||
const val PLACEHOLDER_BASE: Int = 0xE000
|
||||
const val PLACEHOLDER_LIMIT: Int = 0xF8FF - PLACEHOLDER_BASE
|
||||
|
||||
// Texts shorter than this, or with no letter codepoints (emoji-only, punctuation), are skipped
|
||||
// before any ML Kit work — language identification is unreliable on them.
|
||||
private const val MIN_TRANSLATABLE_LENGTH = 4
|
||||
|
||||
val lnRegex: Pattern = Pattern.compile("\\blnbc[a-z0-9]+\\b", Pattern.CASE_INSENSITIVE)
|
||||
val tagRegex: Pattern =
|
||||
Pattern.compile(
|
||||
"(nostr:)?@?(nsec1|npub1|nevent1|naddr1|note1|nprofile1|nrelay1)([qpzry9x8gf2tvdw0s3jn54khce6mua7l]+)",
|
||||
Pattern.CASE_INSENSITIVE,
|
||||
)
|
||||
|
||||
// Legacy NIP-08 positional references like #[0]. Translators tend to insert a space inside the
|
||||
// brackets ("# [0]"), so we shield them via the placeholder dictionary.
|
||||
val nip08RefRegex: Pattern = Pattern.compile("#\\[\\d+]")
|
||||
|
||||
fun isWorthTranslating(text: String): Boolean {
|
||||
if (text.length < MIN_TRANSLATABLE_LENGTH) return false
|
||||
for (cp in text.codePoints()) {
|
||||
if (Character.isLetter(cp)) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
fun build(text: String): Map<String, String> {
|
||||
val dict = LinkedHashMap<String, String>()
|
||||
var counter = 0
|
||||
|
||||
fun addUnique(value: String) {
|
||||
if (value.isEmpty()) return
|
||||
if (counter > PLACEHOLDER_LIMIT) return
|
||||
if (dict.containsValue(value)) return
|
||||
dict[placeholder(counter++)] = value
|
||||
}
|
||||
|
||||
val lnMatcher = lnRegex.matcher(text)
|
||||
while (lnMatcher.find()) addUnique(lnMatcher.group())
|
||||
|
||||
val tagMatcher = tagRegex.matcher(text)
|
||||
while (tagMatcher.find()) addUnique(tagMatcher.group())
|
||||
|
||||
val nip08Matcher = nip08RefRegex.matcher(text)
|
||||
while (nip08Matcher.find()) addUnique(nip08Matcher.group())
|
||||
|
||||
for (url in UrlDetector(text).detect()) {
|
||||
val original = url.originalUrl
|
||||
// The URL detector greedily includes Chinese full-width punctuation; skip those false hits.
|
||||
if (original.contains(',') || original.contains('。')) continue
|
||||
addUnique(original)
|
||||
}
|
||||
|
||||
return dict
|
||||
}
|
||||
|
||||
fun encode(
|
||||
text: String,
|
||||
dict: Map<String, String>,
|
||||
): String {
|
||||
if (dict.isEmpty()) return text
|
||||
var newText = text
|
||||
// Replace longest values first so a URL prefix never clobbers a longer URL or tag.
|
||||
for ((token, original) in dict.entries.sortedByDescending { it.value.length }) {
|
||||
newText = newText.replace(original, token, ignoreCase = false)
|
||||
}
|
||||
return newText
|
||||
}
|
||||
|
||||
fun decode(
|
||||
text: String?,
|
||||
dict: Map<String, String>,
|
||||
): String? {
|
||||
if (text == null || dict.isEmpty()) return text
|
||||
var newText: String = text
|
||||
for ((token, original) in dict) {
|
||||
newText = newText.replace(token, original, ignoreCase = false)
|
||||
}
|
||||
return newText
|
||||
}
|
||||
|
||||
fun placeholder(index: Int): String {
|
||||
require(index in 0..PLACEHOLDER_LIMIT) { "placeholder index $index out of range" }
|
||||
return String(Character.toChars(PLACEHOLDER_BASE + index))
|
||||
}
|
||||
}
|
||||
+283
@@ -0,0 +1,283 @@
|
||||
/*
|
||||
* Copyright (c) 2025 Vitor Pamplona
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to use,
|
||||
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
|
||||
* Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
|
||||
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
package com.vitorpamplona.amethyst.service.lang
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertNotEquals
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Assert.fail
|
||||
import org.junit.Test
|
||||
|
||||
class TranslationDictionaryTest {
|
||||
// ----- isWorthTranslating -----
|
||||
|
||||
@Test
|
||||
fun `short text is not worth translating`() {
|
||||
assertFalse(TranslationDictionary.isWorthTranslating(""))
|
||||
assertFalse(TranslationDictionary.isWorthTranslating("a"))
|
||||
assertFalse(TranslationDictionary.isWorthTranslating("ab"))
|
||||
assertFalse(TranslationDictionary.isWorthTranslating("abc"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `letterless text is not worth translating`() {
|
||||
assertFalse(TranslationDictionary.isWorthTranslating("123456"))
|
||||
assertFalse(TranslationDictionary.isWorthTranslating("!!!!!!"))
|
||||
assertFalse(TranslationDictionary.isWorthTranslating(" "))
|
||||
// Emoji-only.
|
||||
assertFalse(TranslationDictionary.isWorthTranslating("😊😊😊"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `text with at least one letter is worth translating`() {
|
||||
assertTrue(TranslationDictionary.isWorthTranslating("Hello"))
|
||||
assertTrue(TranslationDictionary.isWorthTranslating("a123"))
|
||||
assertTrue(TranslationDictionary.isWorthTranslating("你好世界"))
|
||||
// Mixed emoji + letters.
|
||||
assertTrue(TranslationDictionary.isWorthTranslating("😊 hi"))
|
||||
}
|
||||
|
||||
// ----- placeholder -----
|
||||
|
||||
@Test
|
||||
fun `placeholder is a single Unicode Private Use Area codepoint`() {
|
||||
val p0 = TranslationDictionary.placeholder(0)
|
||||
val p1 = TranslationDictionary.placeholder(1)
|
||||
assertEquals(1, p0.codePointCount(0, p0.length))
|
||||
assertEquals(1, p1.codePointCount(0, p1.length))
|
||||
assertEquals(0xE000, p0.codePointAt(0))
|
||||
assertEquals(0xE001, p1.codePointAt(0))
|
||||
assertNotEquals(p0, p1)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `placeholder rejects out of range index`() {
|
||||
try {
|
||||
TranslationDictionary.placeholder(-1)
|
||||
fail("expected IllegalArgumentException for negative index")
|
||||
} catch (_: IllegalArgumentException) {
|
||||
// expected
|
||||
}
|
||||
try {
|
||||
TranslationDictionary.placeholder(TranslationDictionary.PLACEHOLDER_LIMIT + 1)
|
||||
fail("expected IllegalArgumentException for index past limit")
|
||||
} catch (_: IllegalArgumentException) {
|
||||
// expected
|
||||
}
|
||||
}
|
||||
|
||||
// ----- build -----
|
||||
|
||||
@Test
|
||||
fun `build empty dictionary for plain text`() {
|
||||
val dict = TranslationDictionary.build("Just plain text with no special tokens")
|
||||
assertTrue(dict.isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `build picks up a single URL`() {
|
||||
val text = "Have you seen this https://t.me/mygroup yet?"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
assertEquals(1, dict.size)
|
||||
assertTrue("dict should contain the URL value", dict.containsValue("https://t.me/mygroup"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `build picks up nostr NIP-19 references`() {
|
||||
val text = "see nostr:nevent1qqs0tsw8hjacs4fppgdg7f5yhgwwfkyua4xcs3re9wwkpkk2qeu6mhql22rcy here"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
assertEquals(1, dict.size)
|
||||
assertTrue(dict.containsValue("nostr:nevent1qqs0tsw8hjacs4fppgdg7f5yhgwwfkyua4xcs3re9wwkpkk2qeu6mhql22rcy"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `build picks up Lightning invoices`() {
|
||||
val invoice =
|
||||
"lnbc12u1p3lvjeupp5a5ecgp45k6pa8tu7rnkgzfuwdy3l5ylv3k5tdzrg4cr8rj2f364sdq5g9kxy7fqd9h8vmmfvdjs"
|
||||
val dict = TranslationDictionary.build("Pay me: $invoice please")
|
||||
assertEquals(1, dict.size)
|
||||
assertTrue(dict.containsValue(invoice))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `build picks up legacy NIP-08 positional references`() {
|
||||
val text = "Have you seen this, #[0]"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
assertEquals(1, dict.size)
|
||||
assertTrue(dict.containsValue("#[0]"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `build deduplicates repeated occurrences of the same value`() {
|
||||
val text = "https://a.com and again https://a.com"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
assertEquals(1, dict.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `build collects multiple distinct tokens`() {
|
||||
val text =
|
||||
"ln: lnbc12u1p3lvjeupp5a5ecgp45k6pa8tu7rnkgzfuwdy3l5ylv3 url: https://a.com " +
|
||||
"ref: nostr:nevent1qqsabcdefghjklmnpqrstuvwxyz023456789 nip08: #[0]"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
// We expect at least one entry per category. Exact count depends on the regexes' bech32-charset
|
||||
// truncation behaviour; the contract we care about is that each distinct kind is captured.
|
||||
assertTrue(dict.values.any { it.startsWith("lnbc") })
|
||||
assertTrue("https://a.com" in dict.values)
|
||||
assertTrue(dict.values.any { it.startsWith("nostr:nevent1") })
|
||||
assertTrue("#[0]" in dict.values)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `build rejects URLs with Chinese full-width punctuation false-positives`() {
|
||||
// The URL detector greedily includes , and 。 — those substrings are not real URLs.
|
||||
val text = "看 http://x.com,再见。"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
for (value in dict.values) {
|
||||
assertFalse("URL with , or 。 should be skipped: $value", value.contains(',') || value.contains('。'))
|
||||
}
|
||||
}
|
||||
|
||||
// ----- encode / decode round-trip -----
|
||||
|
||||
@Test
|
||||
fun `encode replaces dictionary values with placeholders and decode restores them`() {
|
||||
val text = "Have you seen this https://t.me/mygroup ?"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
|
||||
assertFalse("URL must be removed from encoded text", encoded.contains("https://t.me/mygroup"))
|
||||
assertTrue("encoded text must contain the placeholder", encoded.codePoints().anyMatch { it in 0xE000..0xF8FF })
|
||||
|
||||
val decoded = TranslationDictionary.decode(encoded, dict)
|
||||
assertEquals(text, decoded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `round-trip preserves nostr references through a simulated translation`() {
|
||||
val text = "Have you seen this, #[0] and nostr:nevent1qqsabcdefgh023456?"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
|
||||
// Simulate a translator: rewrite the surrounding English to Portuguese, but pass placeholders through unchanged.
|
||||
val translated = encoded.replace("Have you seen this", "Você já viu isso").replace("and", "e")
|
||||
|
||||
val decoded = TranslationDictionary.decode(translated, dict)!!
|
||||
assertTrue("decoded must contain #[0]", decoded.contains("#[0]"))
|
||||
assertTrue("decoded must contain the nostr ref", decoded.contains("nostr:nevent1qqsabcdefgh023456"))
|
||||
assertFalse("decoded must not leak placeholder codepoints", decoded.codePoints().anyMatch { it in 0xE000..0xF8FF })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `round-trip preserves multiple URLs of differing lengths`() {
|
||||
val text =
|
||||
"short https://a.co and " +
|
||||
"long https://i.imgur.com/asdEZ3QPswadfj2389rioasdjf9834riofaj9834aKLL.jpg end"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
val decoded = TranslationDictionary.decode(encoded, dict)
|
||||
assertEquals(text, decoded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `encode replaces longer values first to avoid prefix collisions`() {
|
||||
// If "https://a.co" was replaced before "https://a.co/long", the longer URL would be partially clobbered.
|
||||
val text = "long https://a.co/long short https://a.co end"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
// Both URLs must be fully replaced — no leftover http:// fragments.
|
||||
assertFalse("no leftover URL fragment in encoded text", encoded.contains("https://"))
|
||||
val decoded = TranslationDictionary.decode(encoded, dict)
|
||||
assertEquals(text, decoded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decode does not corrupt user text containing the old B0 C0 A0 placeholders`() {
|
||||
// Regression for the pre-rewrite bug: old placeholders "B0", "C0", "A0" collided with arbitrary
|
||||
// user content. The new PUA placeholders are invisible codepoints that cannot occur in normal text,
|
||||
// so a sentence mentioning "B0" or "C0" should round-trip unchanged when there's nothing to replace.
|
||||
val text = "Pricing tier B0 vs C0 vs A0 — see https://docs.example.com/tiers"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
val decoded = TranslationDictionary.decode(encoded, dict)!!
|
||||
assertTrue(decoded.contains("B0"))
|
||||
assertTrue(decoded.contains("C0"))
|
||||
assertTrue(decoded.contains("A0"))
|
||||
assertEquals(text, decoded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `case sensitive replacement preserves user text that differs only in case`() {
|
||||
// The pre-rewrite implementation used ignoreCase=true, which could mangle user text that looked
|
||||
// like a URL placeholder in a different case. With case-sensitive replacement this can't happen.
|
||||
val text = "Visit HTTPS://A.COM/Path then revisit https://a.com/Path"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
val decoded = TranslationDictionary.decode(encoded, dict)
|
||||
assertEquals(text, decoded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `encode is no-op when dictionary is empty`() {
|
||||
val text = "Plain text without anything special"
|
||||
assertEquals(text, TranslationDictionary.encode(text, emptyMap()))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decode handles null input`() {
|
||||
assertNull(TranslationDictionary.decode(null, mapOf("a" to "b")))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decode is no-op when dictionary is empty`() {
|
||||
val text = "anything goes"
|
||||
assertEquals(text, TranslationDictionary.decode(text, emptyMap()))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `mixed content from real-world test cases round-trips`() {
|
||||
val text =
|
||||
"Hi there! \n How are you doing? \n https://i.imgur.com/asdEZ3QPswadfj2389rioasdjf9834riofaj9834aKLL.jpg"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
val decoded = TranslationDictionary.decode(encoded, dict)
|
||||
assertEquals(text, decoded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `complex real-world post round-trips`() {
|
||||
// Mirrors TranslationsTest#testHttp: URL + emoji + multiple NIP-19 references.
|
||||
val text =
|
||||
"https://m.primal.net/MdDd.png \nRunning... 😁 " +
|
||||
"nostr:npub126ntw5mnermmj0znhjhgdk8lh2af72sm8qfzq48umdlnhaj9kuns3le9ll " +
|
||||
"nostr:npub1getal6ykt05fsz5nqu4uld09nfj3y3qxmv8crys4aeut53unfvlqr80nfm"
|
||||
val dict = TranslationDictionary.build(text)
|
||||
val encoded = TranslationDictionary.encode(text, dict)
|
||||
val decoded = TranslationDictionary.decode(encoded, dict)
|
||||
assertEquals(text, decoded)
|
||||
// And every special token must have been replaced in the encoded form.
|
||||
assertFalse(encoded.contains("https://m.primal.net/MdDd.png"))
|
||||
assertFalse(encoded.contains("nostr:npub126ntw5mnermmj0znhjhgdk8lh2af72sm8qfzq48umdlnhaj9kuns3le9ll"))
|
||||
assertFalse(encoded.contains("nostr:npub1getal6ykt05fsz5nqu4uld09nfj3y3qxmv8crys4aeut53unfvlqr80nfm"))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user