feat(quartz): FsEventStore NIP-50 full-text search (step 7)

Adds idx/fts/<token>/<ts>-<id> hardlinks and an FTS-driven query path.

- FsSearchTokenizer: lowercase + Unicode-aware split on non letter-or-
  digit, matching SQLite FTS5's unicode61 default closely enough that
  the same call indexes content and parses queries (any drift cancels).
  Tokens capped at 100 chars to keep filenames under FS limits.
- FsLayout: idxFts + ftsEntry / ftsTokenDir helpers; skeleton dir.
- FsIndexer.pathsFor: when event implements SearchableEvent, emits one
  hardlink per unique tokenised word — so insert/delete maintenance
  rides the existing link/unlink path. Eviction (replaceable swap),
  NIP-09 cascade and NIP-62 vanish all clean up FTS for free.
- FsQueryPlanner: when filter.search is non-blank, drives by FTS.
  Tokenises the query, walks each idx/fts/<token>/ listing into a
  HashMap<id, ts>, and intersects smallest-first (AND across tokens —
  matching SQLite FTS5 default MATCH semantics). Output sorted by
  createdAt DESC. Other Filter fields (kinds, authors, tags, since /
  until) still apply via Filter.match post-filter.

Tests: 16 new in FsSearchTest covering tokenizer (whitespace, case,
unicode, punctuation, empty), index maintenance (entries created,
non-searchable kinds skipped, delete unlinks), and query semantics
(single token, AND of tokens, ordering, limit, kind/author compose,
no-match, blank string ignored, reopen). 86 fs tests green.

quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt

@@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy
 import com.vitorpamplona.quartz.nip01Core.store.sqlite.IndexingStrategy
 import com.vitorpamplona.quartz.nip01Core.store.sqlite.TagNameValueHasher
 import com.vitorpamplona.quartz.nip40Expiration.expiration
+import com.vitorpamplona.quartz.nip50Search.SearchableEvent
 import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
 import java.nio.file.FileAlreadyExistsException
 import java.nio.file.Files
@@ -67,6 +68,11 @@ internal class FsIndexer(
         if (exp != null && exp > 0) {
             out.add(layout.expirationEntry(exp, event.id))
         }
+        if (event is SearchableEvent) {
+            for (token in FsSearchTokenizer.tokenize(event.indexableContent())) {
+                out.add(layout.ftsEntry(token, event.createdAt, event.id))
+            }
+        }
         return out
     }
 

quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt

@@ -46,6 +46,7 @@ internal class FsLayout(
     val idxOwner: Path = idx.resolve(IDX_OWNER)
     val idxTag: Path = idx.resolve(IDX_TAG)
     val idxExpiresAt: Path = idx.resolve(IDX_EXPIRES_AT)
+    val idxFts: Path = idx.resolve(IDX_FTS)
     val replaceable: Path = root.resolve(REPLACEABLE_DIR)
     val addressable: Path = root.resolve(ADDRESSABLE_DIR)
     val tombstones: Path = root.resolve(TOMBSTONES_DIR)
@@ -89,6 +90,15 @@ internal class FsLayout(
         id: HexKey,
     ): Path = idxExpiresAt.resolve(entryName(exp, id))
 
+    /** NIP-50 FTS entry: `idx/fts/<token>/<ts>-<id>`, hardlink to canonical. */
+    fun ftsEntry(
+        token: String,
+        ts: Long,
+        id: HexKey,
+    ): Path = idxFts.resolve(token).resolve(entryName(ts, id))
+
+    fun ftsTokenDir(token: String): Path = idxFts.resolve(token)
+
     /** Directory that holds every indexed value for a tag name. */
     fun tagDir(name: String): Path = idxTag.resolve(name)
 
@@ -139,6 +149,7 @@ internal class FsLayout(
         Files.createDirectories(idxOwner)
         Files.createDirectories(idxTag)
         Files.createDirectories(idxExpiresAt)
+        Files.createDirectories(idxFts)
         Files.createDirectories(replaceable)
         Files.createDirectories(addressable)
         Files.createDirectories(tombstonesId)
@@ -174,6 +185,7 @@ internal class FsLayout(
         const val IDX_OWNER = "owner"
         const val IDX_TAG = "tag"
         const val IDX_EXPIRES_AT = "expires_at"
+        const val IDX_FTS = "fts"
         const val REPLACEABLE_DIR = "replaceable"
         const val ADDRESSABLE_DIR = "addressable"
         const val TOMBSTONES_DIR = "tombstones"

quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt

@@ -59,6 +59,13 @@ internal class FsQueryPlanner(
             return idsDriver(ids)
         }
 
+        // NIP-50 search drives by FTS first when present so the AND
+        // intersection of token sets is the smallest possible candidate
+        // pool. All other predicates are post-filtered via Filter.match.
+        filter.search?.takeIf { it.isNotBlank() }?.let { search ->
+            return ftsDriver(search)
+        }
+
         firstTagKey(filter)?.let { (name, values) ->
             return mergeDesc(values.map { v -> walkDir(layout.tagValueDir(name, hasher.hash(name, v))) })
         }
@@ -107,6 +114,50 @@ internal class FsQueryPlanner(
             yieldAll(mergeDesc(subs.map { walkDir(it) }))
         }
 
+    /**
+     * NIP-50 driver. Tokenises the search string, walks each
+     * `idx/fts/<token>/` listing, and intersects them by id (AND across
+     * tokens — matching SQLite FTS5 default semantics). Output is sorted
+     * by `createdAt` DESC.
+     */
+    private fun ftsDriver(search: String): Sequence<Candidate> =
+        sequence {
+            val tokens = FsSearchTokenizer.tokenize(search)
+            if (tokens.isEmpty()) return@sequence
+            // Materialise each token's listing, then intersect by id.
+            val perToken =
+                tokens.map { token ->
+                    val map = HashMap<HexKey, Long>()
+                    val dir = layout.ftsTokenDir(token)
+                    if (Files.isDirectory(dir)) {
+                        Files.list(dir).use { stream ->
+                            for (entry in stream) {
+                                val parsed = FsLayout.parseEntry(entry.fileName.toString()) ?: continue
+                                map[parsed.second] = parsed.first
+                            }
+                        }
+                    }
+                    map
+                }
+            if (perToken.any { it.isEmpty() }) return@sequence
+            // Start from the smallest set, intersect successively.
+            val sorted = perToken.sortedBy { it.size }
+            var acc = sorted[0]
+            for (i in 1 until sorted.size) {
+                val next = sorted[i]
+                val merged = HashMap<HexKey, Long>(acc.size)
+                for ((id, ts) in acc) {
+                    if (next.containsKey(id)) merged[id] = ts
+                }
+                acc = merged
+                if (acc.isEmpty()) return@sequence
+            }
+            acc.entries
+                .map { Candidate(it.value, it.key) }
+                .sortedByDescending { it.createdAt }
+                .forEach { yield(it) }
+        }
+
     // ---- index directory walker --------------------------------------
 
     private fun walkDir(dir: Path): Sequence<Candidate> =

quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTokenizer.kt

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.quartz.nip01Core.store.fs
+
+/**
+ * NIP-50 search tokenizer for the file-backed store.
+ *
+ * Approximates SQLite FTS5's `unicode61` default tokenizer:
+ *  - Splits on any non letter-or-digit character (Unicode aware via
+ *    `Char.isLetterOrDigit`).
+ *  - Lowercases each emitted token.
+ *  - Drops empty tokens.
+ *  - Truncates pathologically long tokens to keep filenames within
+ *    typical filesystem limits (255 bytes on ext4 / NTFS / APFS).
+ *
+ * The same function is called for indexing and querying, so any
+ * behaviour drift cancels out.
+ */
+internal object FsSearchTokenizer {
+    /** Returns the unique set of search tokens contained in [content]. */
+    fun tokenize(content: String): Set<String> {
+        if (content.isEmpty()) return emptySet()
+        val out = HashSet<String>()
+        val sb = StringBuilder()
+        for (ch in content) {
+            if (ch.isLetterOrDigit()) {
+                sb.append(ch.lowercaseChar())
+            } else if (sb.isNotEmpty()) {
+                emit(sb, out)
+            }
+        }
+        if (sb.isNotEmpty()) emit(sb, out)
+        return out
+    }
+
+    private fun emit(
+        sb: StringBuilder,
+        out: HashSet<String>,
+    ) {
+        val token = if (sb.length <= MAX_TOKEN_LEN) sb.toString() else sb.substring(0, MAX_TOKEN_LEN)
+        out.add(token)
+        sb.setLength(0)
+    }
+
+    /**
+     * Cap on token length. UTF-8 of an all-ASCII string at this length is
+     * 200 bytes — well under the 255-byte path-component limit common to
+     * ext4, APFS and NTFS even when wrapped in the `<ts>-<id>` filename
+     * pattern (76 extra bytes).
+     */
+    private const val MAX_TOKEN_LEN = 100
+}

quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt

@@ -0,0 +1,258 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.quartz.nip01Core.store.fs
+
+import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent
+import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
+import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync
+import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent
+import com.vitorpamplona.quartz.utils.Secp256k1Instance
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.exists
+import kotlin.io.path.listDirectoryEntries
+import kotlin.test.AfterTest
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+class FsSearchTest {
+    private val signer = NostrSignerSync()
+    private lateinit var root: Path
+    private lateinit var store: FsEventStore
+
+    @BeforeTest
+    fun setup() {
+        Secp256k1Instance
+        root = Files.createTempDirectory("fs-search-")
+        store = FsEventStore(root)
+    }
+
+    @AfterTest
+    fun tearDown() {
+        store.close()
+        if (root.exists()) {
+            Files.walk(root).use { it.sorted(Comparator.reverseOrder()).forEach { p -> Files.deleteIfExists(p) } }
+        }
+    }
+
+    private fun note(
+        body: String,
+        ts: Long,
+    ) = signer.sign<TextNoteEvent>(TextNoteEvent.build(body, createdAt = ts))
+
+    // ------------------------------------------------------------------
+    // Tokenizer behaviour
+    // ------------------------------------------------------------------
+
+    @Test
+    fun `tokenizer splits on whitespace and punctuation`() {
+        assertEquals(setOf("hello", "world"), FsSearchTokenizer.tokenize("hello, world!"))
+    }
+
+    @Test
+    fun `tokenizer is case insensitive`() {
+        assertEquals(FsSearchTokenizer.tokenize("Bitcoin"), FsSearchTokenizer.tokenize("BITCOIN"))
+        assertEquals(FsSearchTokenizer.tokenize("Bitcoin"), FsSearchTokenizer.tokenize("bitcoin"))
+    }
+
+    @Test
+    fun `tokenizer handles empty and punctuation-only strings`() {
+        assertEquals(emptySet<String>(), FsSearchTokenizer.tokenize(""))
+        assertEquals(emptySet<String>(), FsSearchTokenizer.tokenize("..."))
+        assertEquals(emptySet<String>(), FsSearchTokenizer.tokenize("   "))
+    }
+
+    @Test
+    fun `tokenizer keeps unicode letters`() {
+        assertEquals(setOf("café", "über"), FsSearchTokenizer.tokenize("Café Über"))
+    }
+
+    // ------------------------------------------------------------------
+    // Index maintenance
+    // ------------------------------------------------------------------
+
+    @Test
+    fun `searchable event creates one fts entry per unique token`() {
+        val n = note("bitcoin nostr bitcoin", ts = 100)
+        store.insert(n)
+
+        val ftsRoot = root.resolve("idx/fts")
+        val tokenDirs = ftsRoot.listDirectoryEntries().map { it.fileName.toString() }.toSet()
+        // TextNoteEvent.indexableContent() prepends a "Subject: " prefix so
+        // we get the content tokens plus the subject ones. What matters is
+        // that each unique token yields exactly one entry under its dir.
+        assertTrue("bitcoin" in tokenDirs)
+        assertTrue("nostr" in tokenDirs)
+        assertEquals(1, ftsRoot.resolve("bitcoin").listDirectoryEntries().size)
+        assertEquals(1, ftsRoot.resolve("nostr").listDirectoryEntries().size)
+    }
+
+    @Test
+    fun `non-searchable event does not produce fts entries`() {
+        val meta =
+            signer.sign<MetadataEvent>(
+                createdAt = 1,
+                kind = MetadataEvent.KIND,
+                tags = emptyArray(),
+                content = "{\"name\":\"vitor\"}",
+            )
+        store.insert(meta)
+
+        val ftsRoot = root.resolve("idx/fts")
+        assertEquals(0, ftsRoot.listDirectoryEntries().size, "MetadataEvent is not SearchableEvent")
+    }
+
+    @Test
+    fun `delete removes fts entries`() {
+        val n = note("bitcoin nostr", ts = 100)
+        store.insert(n)
+        store.delete(n.id)
+
+        val ftsRoot = root.resolve("idx/fts")
+        // Token directories may remain as empty husks.
+        for (tokenDir in ftsRoot.listDirectoryEntries()) {
+            assertEquals(0, tokenDir.listDirectoryEntries().size, "token entry leaked: $tokenDir")
+        }
+    }
+
+    // ------------------------------------------------------------------
+    // Search query semantics
+    // ------------------------------------------------------------------
+
+    @Test
+    fun `single-token search returns the matching event`() {
+        val a = note("bitcoin is fun", ts = 1)
+        val b = note("nostr is also fun", ts = 2)
+        store.insert(a)
+        store.insert(b)
+
+        val got = store.query<TextNoteEvent>(Filter(search = "bitcoin"))
+        assertEquals(listOf(a.id), got.map { it.id })
+    }
+
+    @Test
+    fun `multi-token search is AND across tokens`() {
+        val a = note("bitcoin only", ts = 1)
+        val b = note("nostr only", ts = 2)
+        val c = note("bitcoin and nostr", ts = 3)
+        store.insert(a)
+        store.insert(b)
+        store.insert(c)
+
+        val got = store.query<TextNoteEvent>(Filter(search = "bitcoin nostr"))
+        assertEquals(listOf(c.id), got.map { it.id }, "AND semantics: only the doc with both tokens matches")
+    }
+
+    @Test
+    fun `search results are ordered by createdAt DESC`() {
+        val older = note("bitcoin first", ts = 10)
+        val newer = note("bitcoin again", ts = 20)
+        store.insert(older)
+        store.insert(newer)
+
+        val got = store.query<TextNoteEvent>(Filter(search = "bitcoin"))
+        assertEquals(listOf(newer.id, older.id), got.map { it.id })
+    }
+
+    @Test
+    fun `search respects limit`() {
+        repeat(5) { i -> store.insert(note("bitcoin doc $i", ts = i.toLong() + 1)) }
+        val got = store.query<TextNoteEvent>(Filter(search = "bitcoin", limit = 2))
+        assertEquals(2, got.size)
+    }
+
+    @Test
+    fun `search composes with kinds and authors via post-filter`() {
+        val match = note("bitcoin maximalism", ts = 5)
+        store.insert(match)
+
+        val got =
+            store.query<TextNoteEvent>(
+                Filter(search = "bitcoin", kinds = listOf(1), authors = listOf(signer.pubKey)),
+            )
+        assertEquals(listOf(match.id), got.map { it.id })
+
+        val miss =
+            store.query<TextNoteEvent>(
+                Filter(search = "bitcoin", kinds = listOf(2)),
+            )
+        assertEquals(emptyList(), miss.map { it.id })
+    }
+
+    @Test
+    fun `search with no matching token returns empty`() {
+        store.insert(note("nostr only", ts = 1))
+        assertEquals(
+            emptyList(),
+            store.query<TextNoteEvent>(Filter(search = "bitcoin")).map { it.id },
+        )
+    }
+
+    @Test
+    fun `blank search string is ignored`() {
+        val a = note("anything", ts = 1)
+        store.insert(a)
+        // Blank search shouldn't drive by FTS — the planner falls through
+        // to all-kinds, and the event surfaces.
+        val got = store.query<TextNoteEvent>(Filter(search = "   "))
+        assertEquals(listOf(a.id), got.map { it.id })
+    }
+
+    @Test
+    fun `search survives reopen`() {
+        val n = note("persistent token", ts = 100)
+        store.insert(n)
+        store.close()
+
+        val reopened = FsEventStore(root)
+        try {
+            val got = reopened.query<TextNoteEvent>(Filter(search = "persistent"))
+            assertEquals(listOf(n.id), got.map { it.id })
+        } finally {
+            reopened.close()
+        }
+    }
+
+    // ------------------------------------------------------------------
+    // Maintenance under replaceable / deletion / vanish
+    // ------------------------------------------------------------------
+
+    @Test
+    fun `fts entry is unlinked when event is deleted`() {
+        val n = note("unique-token-zzz", ts = 1)
+        store.insert(n)
+        assertTrue(root.resolve("idx/fts/unique").exists())
+        assertTrue(root.resolve("idx/fts/token").exists())
+        assertTrue(root.resolve("idx/fts/zzz").exists())
+
+        store.delete(n.id)
+        assertFalse(
+            root.resolve("idx/fts/zzz").let { it.exists() && it.listDirectoryEntries().isNotEmpty() },
+            "zzz token entry should be unlinked",
+        )
+
+        // And a search no longer finds it.
+        assertEquals(emptyList(), store.query<TextNoteEvent>(Filter(search = "zzz")).map { it.id })
+    }
+}