diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt index f53e0aae7..71558fa75 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy import com.vitorpamplona.quartz.nip01Core.store.sqlite.TagNameValueHasher import com.vitorpamplona.quartz.nip40Expiration.expiration +import com.vitorpamplona.quartz.nip50Search.SearchableEvent import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import java.nio.file.FileAlreadyExistsException import java.nio.file.Files @@ -67,6 +68,11 @@ internal class FsIndexer( if (exp != null && exp > 0) { out.add(layout.expirationEntry(exp, event.id)) } + if (event is SearchableEvent) { + for (token in FsSearchTokenizer.tokenize(event.indexableContent())) { + out.add(layout.ftsEntry(token, event.createdAt, event.id)) + } + } return out } diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt index 7353e74e6..4964c6abf 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt @@ -46,6 +46,7 @@ internal class FsLayout( val idxOwner: Path = idx.resolve(IDX_OWNER) val idxTag: Path = idx.resolve(IDX_TAG) val idxExpiresAt: Path = idx.resolve(IDX_EXPIRES_AT) + val idxFts: Path = idx.resolve(IDX_FTS) val replaceable: Path = root.resolve(REPLACEABLE_DIR) val addressable: Path = root.resolve(ADDRESSABLE_DIR) val tombstones: Path = root.resolve(TOMBSTONES_DIR) @@ -89,6 +90,15 @@ internal class FsLayout( id: HexKey, ): Path = idxExpiresAt.resolve(entryName(exp, id)) + /** NIP-50 FTS entry: `idx/fts//-`, hardlink to canonical. */ + fun ftsEntry( + token: String, + ts: Long, + id: HexKey, + ): Path = idxFts.resolve(token).resolve(entryName(ts, id)) + + fun ftsTokenDir(token: String): Path = idxFts.resolve(token) + /** Directory that holds every indexed value for a tag name. */ fun tagDir(name: String): Path = idxTag.resolve(name) @@ -139,6 +149,7 @@ internal class FsLayout( Files.createDirectories(idxOwner) Files.createDirectories(idxTag) Files.createDirectories(idxExpiresAt) + Files.createDirectories(idxFts) Files.createDirectories(replaceable) Files.createDirectories(addressable) Files.createDirectories(tombstonesId) @@ -174,6 +185,7 @@ internal class FsLayout( const val IDX_OWNER = "owner" const val IDX_TAG = "tag" const val IDX_EXPIRES_AT = "expires_at" + const val IDX_FTS = "fts" const val REPLACEABLE_DIR = "replaceable" const val ADDRESSABLE_DIR = "addressable" const val TOMBSTONES_DIR = "tombstones" diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt index 9f2bec811..380fdecf0 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt @@ -59,6 +59,13 @@ internal class FsQueryPlanner( return idsDriver(ids) } + // NIP-50 search drives by FTS first when present so the AND + // intersection of token sets is the smallest possible candidate + // pool. All other predicates are post-filtered via Filter.match. + filter.search?.takeIf { it.isNotBlank() }?.let { search -> + return ftsDriver(search) + } + firstTagKey(filter)?.let { (name, values) -> return mergeDesc(values.map { v -> walkDir(layout.tagValueDir(name, hasher.hash(name, v))) }) } @@ -107,6 +114,50 @@ internal class FsQueryPlanner( yieldAll(mergeDesc(subs.map { walkDir(it) })) } + /** + * NIP-50 driver. Tokenises the search string, walks each + * `idx/fts//` listing, and intersects them by id (AND across + * tokens — matching SQLite FTS5 default semantics). Output is sorted + * by `createdAt` DESC. + */ + private fun ftsDriver(search: String): Sequence = + sequence { + val tokens = FsSearchTokenizer.tokenize(search) + if (tokens.isEmpty()) return@sequence + // Materialise each token's listing, then intersect by id. + val perToken = + tokens.map { token -> + val map = HashMap() + val dir = layout.ftsTokenDir(token) + if (Files.isDirectory(dir)) { + Files.list(dir).use { stream -> + for (entry in stream) { + val parsed = FsLayout.parseEntry(entry.fileName.toString()) ?: continue + map[parsed.second] = parsed.first + } + } + } + map + } + if (perToken.any { it.isEmpty() }) return@sequence + // Start from the smallest set, intersect successively. + val sorted = perToken.sortedBy { it.size } + var acc = sorted[0] + for (i in 1 until sorted.size) { + val next = sorted[i] + val merged = HashMap(acc.size) + for ((id, ts) in acc) { + if (next.containsKey(id)) merged[id] = ts + } + acc = merged + if (acc.isEmpty()) return@sequence + } + acc.entries + .map { Candidate(it.value, it.key) } + .sortedByDescending { it.createdAt } + .forEach { yield(it) } + } + // ---- index directory walker -------------------------------------- private fun walkDir(dir: Path): Sequence = diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTokenizer.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTokenizer.kt new file mode 100644 index 000000000..a282cd134 --- /dev/null +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTokenizer.kt @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.fs + +/** + * NIP-50 search tokenizer for the file-backed store. + * + * Approximates SQLite FTS5's `unicode61` default tokenizer: + * - Splits on any non letter-or-digit character (Unicode aware via + * `Char.isLetterOrDigit`). + * - Lowercases each emitted token. + * - Drops empty tokens. + * - Truncates pathologically long tokens to keep filenames within + * typical filesystem limits (255 bytes on ext4 / NTFS / APFS). + * + * The same function is called for indexing and querying, so any + * behaviour drift cancels out. + */ +internal object FsSearchTokenizer { + /** Returns the unique set of search tokens contained in [content]. */ + fun tokenize(content: String): Set { + if (content.isEmpty()) return emptySet() + val out = HashSet() + val sb = StringBuilder() + for (ch in content) { + if (ch.isLetterOrDigit()) { + sb.append(ch.lowercaseChar()) + } else if (sb.isNotEmpty()) { + emit(sb, out) + } + } + if (sb.isNotEmpty()) emit(sb, out) + return out + } + + private fun emit( + sb: StringBuilder, + out: HashSet, + ) { + val token = if (sb.length <= MAX_TOKEN_LEN) sb.toString() else sb.substring(0, MAX_TOKEN_LEN) + out.add(token) + sb.setLength(0) + } + + /** + * Cap on token length. UTF-8 of an all-ASCII string at this length is + * 200 bytes — well under the 255-byte path-component limit common to + * ext4, APFS and NTFS even when wrapped in the `-` filename + * pattern (76 extra bytes). + */ + private const val MAX_TOKEN_LEN = 100 +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt new file mode 100644 index 000000000..201efaad3 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.fs + +import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.utils.Secp256k1Instance +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.listDirectoryEntries +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class FsSearchTest { + private val signer = NostrSignerSync() + private lateinit var root: Path + private lateinit var store: FsEventStore + + @BeforeTest + fun setup() { + Secp256k1Instance + root = Files.createTempDirectory("fs-search-") + store = FsEventStore(root) + } + + @AfterTest + fun tearDown() { + store.close() + if (root.exists()) { + Files.walk(root).use { it.sorted(Comparator.reverseOrder()).forEach { p -> Files.deleteIfExists(p) } } + } + } + + private fun note( + body: String, + ts: Long, + ) = signer.sign(TextNoteEvent.build(body, createdAt = ts)) + + // ------------------------------------------------------------------ + // Tokenizer behaviour + // ------------------------------------------------------------------ + + @Test + fun `tokenizer splits on whitespace and punctuation`() { + assertEquals(setOf("hello", "world"), FsSearchTokenizer.tokenize("hello, world!")) + } + + @Test + fun `tokenizer is case insensitive`() { + assertEquals(FsSearchTokenizer.tokenize("Bitcoin"), FsSearchTokenizer.tokenize("BITCOIN")) + assertEquals(FsSearchTokenizer.tokenize("Bitcoin"), FsSearchTokenizer.tokenize("bitcoin")) + } + + @Test + fun `tokenizer handles empty and punctuation-only strings`() { + assertEquals(emptySet(), FsSearchTokenizer.tokenize("")) + assertEquals(emptySet(), FsSearchTokenizer.tokenize("...")) + assertEquals(emptySet(), FsSearchTokenizer.tokenize(" ")) + } + + @Test + fun `tokenizer keeps unicode letters`() { + assertEquals(setOf("café", "über"), FsSearchTokenizer.tokenize("Café Über")) + } + + // ------------------------------------------------------------------ + // Index maintenance + // ------------------------------------------------------------------ + + @Test + fun `searchable event creates one fts entry per unique token`() { + val n = note("bitcoin nostr bitcoin", ts = 100) + store.insert(n) + + val ftsRoot = root.resolve("idx/fts") + val tokenDirs = ftsRoot.listDirectoryEntries().map { it.fileName.toString() }.toSet() + // TextNoteEvent.indexableContent() prepends a "Subject: " prefix so + // we get the content tokens plus the subject ones. What matters is + // that each unique token yields exactly one entry under its dir. + assertTrue("bitcoin" in tokenDirs) + assertTrue("nostr" in tokenDirs) + assertEquals(1, ftsRoot.resolve("bitcoin").listDirectoryEntries().size) + assertEquals(1, ftsRoot.resolve("nostr").listDirectoryEntries().size) + } + + @Test + fun `non-searchable event does not produce fts entries`() { + val meta = + signer.sign( + createdAt = 1, + kind = MetadataEvent.KIND, + tags = emptyArray(), + content = "{\"name\":\"vitor\"}", + ) + store.insert(meta) + + val ftsRoot = root.resolve("idx/fts") + assertEquals(0, ftsRoot.listDirectoryEntries().size, "MetadataEvent is not SearchableEvent") + } + + @Test + fun `delete removes fts entries`() { + val n = note("bitcoin nostr", ts = 100) + store.insert(n) + store.delete(n.id) + + val ftsRoot = root.resolve("idx/fts") + // Token directories may remain as empty husks. + for (tokenDir in ftsRoot.listDirectoryEntries()) { + assertEquals(0, tokenDir.listDirectoryEntries().size, "token entry leaked: $tokenDir") + } + } + + // ------------------------------------------------------------------ + // Search query semantics + // ------------------------------------------------------------------ + + @Test + fun `single-token search returns the matching event`() { + val a = note("bitcoin is fun", ts = 1) + val b = note("nostr is also fun", ts = 2) + store.insert(a) + store.insert(b) + + val got = store.query(Filter(search = "bitcoin")) + assertEquals(listOf(a.id), got.map { it.id }) + } + + @Test + fun `multi-token search is AND across tokens`() { + val a = note("bitcoin only", ts = 1) + val b = note("nostr only", ts = 2) + val c = note("bitcoin and nostr", ts = 3) + store.insert(a) + store.insert(b) + store.insert(c) + + val got = store.query(Filter(search = "bitcoin nostr")) + assertEquals(listOf(c.id), got.map { it.id }, "AND semantics: only the doc with both tokens matches") + } + + @Test + fun `search results are ordered by createdAt DESC`() { + val older = note("bitcoin first", ts = 10) + val newer = note("bitcoin again", ts = 20) + store.insert(older) + store.insert(newer) + + val got = store.query(Filter(search = "bitcoin")) + assertEquals(listOf(newer.id, older.id), got.map { it.id }) + } + + @Test + fun `search respects limit`() { + repeat(5) { i -> store.insert(note("bitcoin doc $i", ts = i.toLong() + 1)) } + val got = store.query(Filter(search = "bitcoin", limit = 2)) + assertEquals(2, got.size) + } + + @Test + fun `search composes with kinds and authors via post-filter`() { + val match = note("bitcoin maximalism", ts = 5) + store.insert(match) + + val got = + store.query( + Filter(search = "bitcoin", kinds = listOf(1), authors = listOf(signer.pubKey)), + ) + assertEquals(listOf(match.id), got.map { it.id }) + + val miss = + store.query( + Filter(search = "bitcoin", kinds = listOf(2)), + ) + assertEquals(emptyList(), miss.map { it.id }) + } + + @Test + fun `search with no matching token returns empty`() { + store.insert(note("nostr only", ts = 1)) + assertEquals( + emptyList(), + store.query(Filter(search = "bitcoin")).map { it.id }, + ) + } + + @Test + fun `blank search string is ignored`() { + val a = note("anything", ts = 1) + store.insert(a) + // Blank search shouldn't drive by FTS — the planner falls through + // to all-kinds, and the event surfaces. + val got = store.query(Filter(search = " ")) + assertEquals(listOf(a.id), got.map { it.id }) + } + + @Test + fun `search survives reopen`() { + val n = note("persistent token", ts = 100) + store.insert(n) + store.close() + + val reopened = FsEventStore(root) + try { + val got = reopened.query(Filter(search = "persistent")) + assertEquals(listOf(n.id), got.map { it.id }) + } finally { + reopened.close() + } + } + + // ------------------------------------------------------------------ + // Maintenance under replaceable / deletion / vanish + // ------------------------------------------------------------------ + + @Test + fun `fts entry is unlinked when event is deleted`() { + val n = note("unique-token-zzz", ts = 1) + store.insert(n) + assertTrue(root.resolve("idx/fts/unique").exists()) + assertTrue(root.resolve("idx/fts/token").exists()) + assertTrue(root.resolve("idx/fts/zzz").exists()) + + store.delete(n.id) + assertFalse( + root.resolve("idx/fts/zzz").let { it.exists() && it.listDirectoryEntries().isNotEmpty() }, + "zzz token entry should be unlinked", + ) + + // And a search no longer finds it. + assertEquals(emptyList(), store.query(Filter(search = "zzz")).map { it.id }) + } +}