From 9fcf85bed083462214285fa66bd0e6ec49ea1e4e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Apr 2026 13:25:30 +0000 Subject: [PATCH] fix(quartz/sqlite): serialise writes via a Room-style connection pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit androidx.sqlite SQLiteConnection is not thread-safe; SQLiteEventStore shared a single lazy connection across all callers, so two coroutines calling insertEvent() at the same time would race on BEGIN IMMEDIATE and the modules' prepared statements, surfacing as "cannot start a transaction within a transaction" or SQLITE_MISUSE. Mirror Room's design: introduce SQLiteConnectionPool with one writer connection guarded by a coroutine Mutex and N reader connections handed out via a Channel-as-semaphore (file-backed DBs only; in-memory DBs share the writer because each ":memory:" connection is a separate DB). Convert IEventStore + SQLiteEventStore + EventStore + FsEventStore + LiveEventStore to suspend, route writes through useWriter and reads through useReader. RelaySession now launches handleEvent / handleCount on its scope. CLI Context helpers and StoreCommands.sweepExpired pick up suspend. Add ParallelInsertTest to lock the behaviour in: 8 coroutines × 200 inserts, parallel reads alongside writes, transaction batches across coroutines, and a reopen smoke test all pass against a file-backed DB. https://claude.ai/code/session_016b5kSSbtDS3Ead6pN3Xqt5 --- .../com/vitorpamplona/amethyst/cli/Context.kt | 24 +- .../amethyst/cli/commands/FeedCommand.kt | 2 +- .../amethyst/cli/commands/ProfileCommands.kt | 2 +- .../amethyst/cli/commands/RelayCommands.kt | 2 +- .../amethyst/cli/commands/StoreCommands.kt | 2 +- .../nip01Core/relay/server/LiveEventStore.kt | 4 +- .../nip01Core/relay/server/RelaySession.kt | 62 +- .../quartz/nip01Core/store/IEventStore.kt | 22 +- .../nip01Core/store/sqlite/EventStore.kt | 28 +- .../nip01Core/store/sqlite/QueryExplainer.kt | 4 +- .../store/sqlite/SQLiteConnectionPool.kt | 135 +++++ .../store/sqlite/SQLiteEventStore.kt | 173 +++--- .../nip01Core/store/sqlite/AssertUtils.kt | 8 +- .../nip01Core/store/sqlite/BasicTest.kt | 12 +- .../nip01Core/store/sqlite/LargeDBTests.kt | 31 +- .../store/sqlite/QueryAssemblerTest.kt | 4 +- .../quartz/nip01Core/store/fs/FsEventStore.kt | 29 +- .../nip01Core/store/fs/FsLockManager.kt | 32 +- .../nip01Core/store/fs/FsDeletionTest.kt | 331 ++++++----- .../nip01Core/store/fs/FsEventStoreTest.kt | 272 ++++----- .../nip01Core/store/fs/FsEventToJsonTest.kt | 107 ++-- .../nip01Core/store/fs/FsExpirationTest.kt | 197 ++++--- .../nip01Core/store/fs/FsMaintenanceTest.kt | 276 ++++----- .../quartz/nip01Core/store/fs/FsParityTest.kt | 509 ++++++++-------- .../quartz/nip01Core/store/fs/FsQueryTest.kt | 544 +++++++++--------- .../quartz/nip01Core/store/fs/FsSearchTest.kt | 335 ++++++----- .../quartz/nip01Core/store/fs/FsSlotsTest.kt | 424 +++++++------- .../quartz/nip01Core/store/fs/FsVanishTest.kt | 238 ++++---- .../store/sqlite/ParallelInsertTest.kt | 205 +++++++ 29 files changed, 2264 insertions(+), 1750 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ParallelInsertTest.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index ab3ab5976..f8c4a21ee 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -173,7 +173,7 @@ class Context( * publish from", which mirrors `User.outboxRelays()` in the * Android app. */ - fun outboxRelays(): Set = + suspend fun outboxRelays(): Set = relaysOf(identity.pubKeyHex)?.writeRelaysNorm()?.takeIf { it.isNotEmpty() }?.toSet() ?: DefaultNIP65RelaySet @@ -181,7 +181,7 @@ class Context( * DM inbox relays (NIP-17 kind:10050) for this account. Falls back * to [DefaultDMRelayList] when no kind:10050 has been seen. */ - fun inboxRelays(): Set = + suspend fun inboxRelays(): Set = dmInboxOf(identity.pubKeyHex)?.relays()?.takeIf { it.isNotEmpty() }?.toSet() ?: DefaultDMRelayList.toSet() @@ -190,12 +190,12 @@ class Context( * back to [outboxRelays] when no kind:10051 has been seen — same * fallback the Android app uses for KeyPackage discovery. */ - fun keyPackageRelays(): Set = + suspend fun keyPackageRelays(): Set = keyPackageRelaysOf(identity.pubKeyHex)?.relays()?.takeIf { it.isNotEmpty() }?.toSet() ?: outboxRelays() /** Union of all three buckets. */ - fun anyRelays(): Set = outboxRelays() + inboxRelays() + keyPackageRelays() + suspend fun anyRelays(): Set = outboxRelays() + inboxRelays() + keyPackageRelays() /** * Seed relays for "look up someone we know nothing about" queries — @@ -208,7 +208,7 @@ class Context( * most reliable place to find a stranger's replaceable events even when * we and they have completely disjoint relay configurations. */ - fun bootstrapRelays(): Set = + suspend fun bootstrapRelays(): Set = buildSet { addAll(anyRelays()) addAll(DefaultNIP65RelaySet) @@ -319,7 +319,7 @@ class Context( * Every event-arrival path in the CLI funnels through this method * so that [store] is the authoritative cache of what Amy has seen. */ - fun verifyAndStore(event: Event): Boolean { + suspend fun verifyAndStore(event: Event): Boolean { if (!event.verify()) { System.err.println("[cli] dropped event ${event.id.take(8)} kind=${event.kind} — bad signature") return false @@ -342,7 +342,7 @@ class Context( * this user. Callers that need a network fetch on miss should fall * back to [drain] explicitly — this helper never hits the network. */ - fun profileOf(pubKey: HexKey): MetadataEvent? = + suspend fun profileOf(pubKey: HexKey): MetadataEvent? = store .query( Filter(authors = listOf(pubKey), kinds = listOf(MetadataEvent.KIND), limit = 1), @@ -352,7 +352,7 @@ class Context( * Latest known kind:10002 advertised relay list (NIP-65) for * [pubKey]. `null` when Amy has never seen one. */ - fun relaysOf(pubKey: HexKey): AdvertisedRelayListEvent? = + suspend fun relaysOf(pubKey: HexKey): AdvertisedRelayListEvent? = store .query( Filter(authors = listOf(pubKey), kinds = listOf(AdvertisedRelayListEvent.KIND), limit = 1), @@ -363,7 +363,7 @@ class Context( * `null` if Amy has never observed one. Useful for follow-graph * lookups without re-hitting relays. */ - fun contactsOf(pubKey: HexKey): ContactListEvent? = + suspend fun contactsOf(pubKey: HexKey): ContactListEvent? = store .query( Filter(authors = listOf(pubKey), kinds = listOf(ContactListEvent.KIND), limit = 1), @@ -374,7 +374,7 @@ class Context( * for [pubKey], or `null` if Amy has never observed one. Used by * `dm send` to resolve where to deliver a wrap. */ - fun dmInboxOf(pubKey: HexKey): ChatMessageRelayListEvent? = + suspend fun dmInboxOf(pubKey: HexKey): ChatMessageRelayListEvent? = store .query( Filter(authors = listOf(pubKey), kinds = listOf(ChatMessageRelayListEvent.KIND), limit = 1), @@ -386,7 +386,7 @@ class Context( * `marmot key-package check` and `marmot await key-package` to * locate where the recipient publishes their KeyPackages. */ - fun keyPackageRelaysOf(pubKey: HexKey): KeyPackageRelayListEvent? = + suspend fun keyPackageRelaysOf(pubKey: HexKey): KeyPackageRelayListEvent? = store .query( Filter(authors = listOf(pubKey), kinds = listOf(KeyPackageRelayListEvent.KIND), limit = 1), @@ -405,7 +405,7 @@ class Context( * we'll still hand back the old list. Commands that care can drain * (which re-populates the cache) or expose a `--refresh` flag. */ - fun cachedRelayListsOf(pubKey: HexKey): RecipientRelayFetcher.Lists? { + suspend fun cachedRelayListsOf(pubKey: HexKey): RecipientRelayFetcher.Lists? { val dm = dmInboxOf(pubKey) val kp = keyPackageRelaysOf(pubKey) val nip65 = relaysOf(pubKey) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/FeedCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/FeedCommand.kt index 87e0c6bfb..ba928622e 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/FeedCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/FeedCommand.kt @@ -142,7 +142,7 @@ object FeedCommand { * an arbitrary `--author` we have no idea where they publish, so we * widen to the bootstrap union. */ - private fun relaysForReadingFeed( + private suspend fun relaysForReadingFeed( ctx: Context, mode: String, ): Set = diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ProfileCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ProfileCommands.kt index 360e34fe9..0b827167f 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ProfileCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ProfileCommands.kt @@ -226,7 +226,7 @@ object ProfileCommands { * else's profile, fall back to the bootstrap union so we still find a * kind:0 even when our relay set and theirs are disjoint. */ - private fun relaysForReadingProfile( + private suspend fun relaysForReadingProfile( ctx: Context, isSelf: Boolean, ): Set = diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt index 256e3b561..aade5d829 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt @@ -137,7 +137,7 @@ object RelayCommands { } } - private fun list(dataDir: DataDir): Int { + private suspend fun list(dataDir: DataDir): Int { val ctx = Context.open(dataDir) try { val self = ctx.identity.pubKeyHex diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt index 4ee0a7ee6..470c42bbf 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/StoreCommands.kt @@ -132,7 +132,7 @@ object StoreCommands { return 0 } - private fun sweepExpired(dataDir: DataDir): Int = + private suspend fun sweepExpired(dataDir: DataDir): Int = withStore(dataDir) { store -> val expiresAtDir = dataDir.eventsDir.toPath().resolve("idx/expires_at") val before = countEntries(expiresAtDir) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/LiveEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/LiveEventStore.kt index 4268dcef3..5e1cf9999 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/LiveEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/LiveEventStore.kt @@ -46,7 +46,7 @@ class LiveEventStore( onBufferOverflow = BufferOverflow.DROP_LATEST, // Default behavior ) - fun insert(event: Event) { + suspend fun insert(event: Event) { store.insert(event) newEventStream.tryEmit(event) } @@ -70,5 +70,5 @@ class LiveEventStore( } } - fun count(filters: List) = store.count(filters) + suspend fun count(filters: List) = store.count(filters) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt index 71a6fb944..dd43f03d4 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt @@ -111,6 +111,36 @@ class RelaySession( } } + private suspend fun handleEvent(cmd: EventCmd) { + val result = policy.accept(cmd) + if (result is PolicyResult.Rejected) { + send(OkMessage(cmd.event.id, false, result.reason)) + return + } + + try { + store.insert(cmd.event) + send(OkMessage(cmd.event.id, true, "")) + } catch (e: Exception) { + send(OkMessage(cmd.event.id, false, e.message ?: e::class.simpleName ?: "unkown error")) + } + } + + private suspend fun handleCount(cmd: CountCmd) { + val result = policy.accept(cmd) + if (result is PolicyResult.Rejected) { + send(ClosedMessage(cmd.queryId, result.reason)) + return + } + + // Policy may rewrite filters to match the user's access level. + val filters = (result as PolicyResult.Accepted).cmd.filters + + val total = store.count(filters) + + send(CountMessage(cmd.queryId, CountResult(total))) + } + // -- NIP-42: AUTH --------------------------------------------------------- private fun handleAuth(cmd: AuthCmd) { val result = policy.accept(cmd) @@ -164,38 +194,6 @@ class RelaySession( } } - // -- NIP-01: EVENT -------------------------------------------------------- - private fun handleEvent(cmd: EventCmd) { - val result = policy.accept(cmd) - if (result is PolicyResult.Rejected) { - send(OkMessage(cmd.event.id, false, result.reason)) - return - } - - try { - store.insert(cmd.event) - send(OkMessage(cmd.event.id, true, "")) - } catch (e: Exception) { - send(OkMessage(cmd.event.id, false, e.message ?: e::class.simpleName ?: "unkown error")) - } - } - - // -- NIP-45: COUNT -------------------------------------------------------- - private fun handleCount(cmd: CountCmd) { - val result = policy.accept(cmd) - if (result is PolicyResult.Rejected) { - send(ClosedMessage(cmd.queryId, result.reason)) - return - } - - // Policy may rewrite filters to match the user's access level. - val filters = (result as PolicyResult.Accepted).cmd.filters - - val total = store.count(filters) - - send(CountMessage(cmd.queryId, CountResult(total))) - } - init { policy.onConnect(::send) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index e9268922d..e70773c29 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -24,37 +24,37 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter interface IEventStore : AutoCloseable { - fun insert(event: Event) + suspend fun insert(event: Event) interface ITransaction { fun insert(event: Event) } - fun transaction(body: ITransaction.() -> Unit) + suspend fun transaction(body: ITransaction.() -> Unit) - fun query(filter: Filter): List + suspend fun query(filter: Filter): List - fun query(filters: List): List + suspend fun query(filters: List): List - fun query( + suspend fun query( filter: Filter, onEach: (T) -> Unit, ) - fun query( + suspend fun query( filters: List, onEach: (T) -> Unit, ) - fun count(filter: Filter): Int + suspend fun count(filter: Filter): Int - fun count(filters: List): Int + suspend fun count(filters: List): Int - fun delete(filter: Filter) + suspend fun delete(filter: Filter) - fun delete(filters: List) + suspend fun delete(filters: List) - fun deleteExpiredEvents() + suspend fun deleteExpiredEvents() override fun close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index 0a861cc9b..7fb287555 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -34,33 +34,37 @@ class EventStore( ) : IEventStore { val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy) - override fun insert(event: Event) = store.insertEvent(event) + override suspend fun insert(event: Event) = store.insertEvent(event) - override fun transaction(body: IEventStore.ITransaction.() -> Unit) = store.transaction(body) + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = store.transaction(body) - override fun query(filter: Filter) = store.query(filter) + override suspend fun query(filter: Filter) = store.query(filter) - override fun query(filters: List) = store.query(filters) + override suspend fun query(filters: List) = store.query(filters) - override fun query( + override suspend fun query( filter: Filter, onEach: (T) -> Unit, ) = store.query(filter, onEach) - override fun query( + override suspend fun query( filters: List, onEach: (T) -> Unit, ) = store.query(filters, onEach) - override fun count(filter: Filter) = store.count(filter) + override suspend fun count(filter: Filter) = store.count(filter) - override fun count(filters: List) = store.count(filters) + override suspend fun count(filters: List) = store.count(filters) - override fun delete(filter: Filter) = store.delete(filter) + override suspend fun delete(filter: Filter) { + store.delete(filter) + } - override fun delete(filters: List) = store.delete(filters) + override suspend fun delete(filters: List) { + store.delete(filters) + } - override fun deleteExpiredEvents() = store.deleteExpiredEvents() + override suspend fun deleteExpiredEvents() = store.deleteExpiredEvents() - override fun close() = store.connection.close() + override fun close() = store.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryExplainer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryExplainer.kt index 4b2d6bf27..841abbea8 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryExplainer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryExplainer.kt @@ -22,10 +22,10 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.SQLiteConnection -fun SQLiteEventStore.explainQuery( +suspend fun SQLiteEventStore.explainQuery( sql: String, args: Array = emptyArray(), -) = connection.explainQuery(sql, args.map { it.toString() }.toTypedArray()) +): String = pool.useReader { it.explainQuery(sql, args.map { a -> a.toString() }.toTypedArray()) } fun SQLiteConnection.explainQuery( sql: String, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt new file mode 100644 index 000000000..e5adc0701 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import androidx.sqlite.SQLiteConnection +import androidx.sqlite.SQLiteDriver +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * Room-style connection pool for an `androidx.sqlite` database. + * + * `androidx.sqlite.SQLiteConnection` is not thread-safe (same contract as + * `sqlite3*` in the C API): a single connection may only be used by one + * thread at a time. Two coroutines hitting the same connection in parallel + * race on `BEGIN IMMEDIATE` and prepared-statement state, which surfaces + * as `SQLITE_ERROR: cannot start a transaction within a transaction` or + * `SQLITE_MISUSE`. + * + * The pool mirrors what Room does: + * + * - **One writer connection**, guarded by a coroutine [Mutex]. SQLite + * only allows a single writer at the file level anyway, so serialising + * writes here costs nothing — it just queues callers cooperatively + * instead of crashing them. + * - **N reader connections**, handed out from a [Channel] that doubles + * as a semaphore. Under WAL (`PRAGMA journal_mode = WAL`) readers run + * in parallel with the writer and with each other. + * + * For in-memory databases (`dbName == null`) every fresh `:memory:` + * connection opens a *separate* database, so the pool degrades to a + * single-connection mode where readers also acquire the writer mutex. + * That still fixes the parallel-insert crash; it just sacrifices reader + * concurrency for an in-memory store. + * + * Lifecycle: + * 1. `init` opens the writer, runs [onConfigure] on it, then [onMigrate] + * so schema exists before any reader sees the file. + * 2. Readers are opened next and each gets [onConfigure] (PRAGMAs are + * per-connection in SQLite — `journal_mode=WAL` is the only + * database-wide one; subsequent connections inherit it). + * 3. [close] drains the reader channel and closes every connection. + */ +class SQLiteConnectionPool( + val driver: SQLiteDriver, + val dbName: String?, + val numReaders: Int = 4, + val onConfigure: (SQLiteConnection) -> Unit = {}, + val onMigrate: (SQLiteConnection) -> Unit = {}, +) : AutoCloseable { + private val isInMemory = dbName == null + + private val writerMutex = Mutex() + val writer: SQLiteConnection + + private val readers: List + private val readerChannel: Channel? + + init { + writer = openConnection() + onMigrate(writer) + + if (isInMemory) { + readers = emptyList() + readerChannel = null + } else { + readers = List(numReaders) { openConnection() } + readerChannel = Channel(numReaders) + readers.forEach { readerChannel.trySend(it) } + } + } + + private fun openConnection(): SQLiteConnection { + val db = driver.open(dbName ?: ":memory:") + onConfigure(db) + return db + } + + /** + * Acquire the writer connection for the duration of [block]. Other + * writers (and, in the in-memory single-connection mode, readers) + * suspend until the lock is released. Cancellation-aware via the + * coroutine [Mutex]. + */ + suspend fun useWriter(block: (SQLiteConnection) -> T): T = + writerMutex.withLock { + block(writer) + } + + /** + * Acquire any free reader connection for [block]. With a file-backed + * DB up to [numReaders] readers run in parallel with the writer + * (WAL). With an in-memory DB this falls back to the writer mutex + * because each `:memory:` connection would be a separate database. + */ + suspend fun useReader(block: (SQLiteConnection) -> T): T { + val ch = + readerChannel + ?: return writerMutex.withLock { block(writer) } + val conn = ch.receive() + try { + return block(conn) + } finally { + // Capacity == numReaders and we own the conn we received, so + // trySend never fails unless the channel was closed mid-flight + // (in which case the connection is being torn down anyway). + ch.trySend(conn) + } + } + + override fun close() { + readerChannel?.close() + readers.forEach { runCatching { it.close() } } + runCatching { writer.close() } + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 60da85fe9..2da5cf49c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -34,24 +34,18 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip40Expiration.isExpired import com.vitorpamplona.quartz.utils.EventFactory -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.IO -import kotlinx.coroutines.withContext class SQLiteEventStore( val driver: SQLiteDriver = BundledSQLiteDriver(), val dbName: String? = "events.db", val relay: NormalizedRelayUrl? = null, val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), + val numReaders: Int = 4, ) { companion object { const val DATABASE_VERSION = 2 } - val connection: SQLiteConnection by lazy { - openAndConfigure() - } - val seedModule = SeedModule() val fullTextSearchModule = FullTextSearchModule() @@ -89,37 +83,44 @@ class SQLiteEventStore( fullTextSearchModule, ) - private fun openAndConfigure(): SQLiteConnection { - val db = driver.open(dbName ?: ":memory:") + val pool: SQLiteConnectionPool by lazy { + SQLiteConnectionPool( + driver = driver, + dbName = dbName, + numReaders = numReaders, + onConfigure = { db -> + // 32MB memory cache (per-connection). + db.execSQL("PRAGMA cache_size=-32000;") - // 32MB memory cache - db.execSQL("PRAGMA cache_size=-32000;") + // Make sure the FKs are sane (per-connection). + db.execSQL("PRAGMA foreign_keys = ON;") - // makes sure the FKs are sane - db.execSQL("PRAGMA foreign_keys = ON;") + // SQLite implements mutations by appending them to a log, + // which it occasionally compacts into the database. This + // is called Write-Ahead Logging (WAL). Setting it on the + // first connection is enough — `journal_mode` is + // database-wide; subsequent connections inherit it. + db.execSQL("PRAGMA journal_mode = WAL;") - // SQLite implements mutations by appending them to a log, which it occasionally - // compacts into the database. This is called Write-Ahead Logging (WAL) - db.execSQL("PRAGMA journal_mode = WAL;") - - // The DB can be corrupted if the OS is shutdown before sync, which generally - // doesn't happen on Android - db.execSQL("PRAGMA synchronous = OFF;") - - val currentVersion = getUserVersion(db) - if (currentVersion == 0) { - db.transaction { - onCreate(this) - setUserVersion(this, DATABASE_VERSION) - } - } else if (currentVersion < DATABASE_VERSION) { - db.transaction { - onUpgrade(this, currentVersion, DATABASE_VERSION) - setUserVersion(this, DATABASE_VERSION) - } - } - - return db + // The DB can be corrupted if the OS shuts down before + // sync, which generally doesn't happen on Android. + db.execSQL("PRAGMA synchronous = OFF;") + }, + onMigrate = { db -> + val currentVersion = getUserVersion(db) + if (currentVersion == 0) { + db.transaction { + onCreate(this) + setUserVersion(this, DATABASE_VERSION) + } + } else if (currentVersion < DATABASE_VERSION) { + db.transaction { + onUpgrade(this, currentVersion, DATABASE_VERSION) + setUserVersion(this, DATABASE_VERSION) + } + } + }, + ) } private fun getUserVersion(db: SQLiteConnection): Int = @@ -159,25 +160,24 @@ class SQLiteEventStore( } } - fun clearDB() { - modules.reversed().forEach { it.deleteAll(connection) } - } - - suspend fun vacuum() { - // VACUUM: Rebuilds the database file, reclaiming unused space - // and reducing fragmentation. - withContext(Dispatchers.IO) { - connection.execSQL("VACUUM") + suspend fun clearDB() = + pool.useWriter { db -> + modules.reversed().forEach { it.deleteAll(db) } } - } - suspend fun analyse() { - // ANALYZE: Collects statistics about tables and indices - // to help the query planner optimize queries. - withContext(Dispatchers.IO) { - connection.execSQL("ANALYZE") + suspend fun vacuum() = + pool.useWriter { db -> + // VACUUM: Rebuilds the database file, reclaiming unused space + // and reducing fragmentation. + db.execSQL("VACUUM") + } + + suspend fun analyse() = + pool.useWriter { db -> + // ANALYZE: Collects statistics about tables and indices + // to help the query planner optimize queries. + db.execSQL("ANALYZE") } - } private fun innerInsertEvent( event: Event, @@ -190,12 +190,14 @@ class SQLiteEventStore( rightToVanishModule.insert(event, relay, headerId, db) } - fun insertEvent(event: Event) { + suspend fun insertEvent(event: Event) { if (event.isExpired()) throw SQLiteException("blocked: Cannot insert an expired event") if (event.kind.isEphemeral()) return - connection.transaction { - innerInsertEvent(event, this) + pool.useWriter { db -> + db.transaction { + innerInsertEvent(event, this) + } } } @@ -210,64 +212,65 @@ class SQLiteEventStore( } } - fun transaction(body: Transaction.() -> Unit) { - connection.transaction { - with(Transaction(this)) { - body() + suspend fun transaction(body: Transaction.() -> Unit) { + pool.useWriter { db -> + db.transaction { + with(Transaction(this)) { + body() + } } } } - fun query(filter: Filter): List = queryBuilder.query(filter, connection) + suspend fun query(filter: Filter): List = pool.useReader { queryBuilder.query(filter, it) } - fun query(filters: List): List = queryBuilder.query(filters, connection) + suspend fun query(filters: List): List = pool.useReader { queryBuilder.query(filters, it) } - fun query( + suspend fun query( filter: Filter, onEach: (T) -> Unit, - ) = queryBuilder.query(filter, connection, onEach) + ) = pool.useReader { queryBuilder.query(filter, it, onEach) } - fun query( + suspend fun query( filters: List, onEach: (T) -> Unit, - ) = queryBuilder.query(filters, connection, onEach) + ) = pool.useReader { queryBuilder.query(filters, it, onEach) } - fun rawQuery(filter: Filter): List = queryBuilder.rawQuery(filter, connection) + suspend fun rawQuery(filter: Filter): List = pool.useReader { queryBuilder.rawQuery(filter, it) } - fun rawQuery(filters: List): List = queryBuilder.rawQuery(filters, connection) + suspend fun rawQuery(filters: List): List = pool.useReader { queryBuilder.rawQuery(filters, it) } - fun rawQuery( + suspend fun rawQuery( filter: Filter, onEach: (RawEvent) -> Unit, - ) = queryBuilder.rawQuery(filter, connection, onEach) + ) = pool.useReader { queryBuilder.rawQuery(filter, it, onEach) } - fun rawQuery( + suspend fun rawQuery( filters: List, onEach: (RawEvent) -> Unit, - ) = queryBuilder.rawQuery(filters, connection, onEach) + ) = pool.useReader { queryBuilder.rawQuery(filters, it, onEach) } - fun planQuery(filter: Filter) = queryBuilder.planQuery(filter, seedModule.hasher(connection), connection) + suspend fun planQuery(filter: Filter) = pool.useReader { queryBuilder.planQuery(filter, seedModule.hasher(it), it) } - fun planQuery(filters: List) = queryBuilder.planQuery(filters, seedModule.hasher(connection), connection) + suspend fun planQuery(filters: List) = pool.useReader { queryBuilder.planQuery(filters, seedModule.hasher(it), it) } - fun count(filter: Filter): Int = queryBuilder.count(filter, connection) + suspend fun count(filter: Filter): Int = pool.useReader { queryBuilder.count(filter, it) } - fun count(filters: List): Int = queryBuilder.count(filters, connection) + suspend fun count(filters: List): Int = pool.useReader { queryBuilder.count(filters, it) } - fun delete(filter: Filter) { - queryBuilder.delete(filter, connection) - } + suspend fun delete(filter: Filter) = pool.useWriter { queryBuilder.delete(filter, it) } - fun delete(filters: List) { - queryBuilder.delete(filters, connection) - } + suspend fun delete(filters: List) = pool.useWriter { queryBuilder.delete(filters, it) } - fun delete(id: HexKey): Int { - connection.execSQL("DELETE FROM event_headers WHERE id = ?", arrayOf(id)) - return connection.changes() - } + suspend fun delete(id: HexKey): Int = + pool.useWriter { db -> + db.execSQL("DELETE FROM event_headers WHERE id = ?", arrayOf(id)) + db.changes() + } - fun deleteExpiredEvents() = expirationModule.deleteExpiredEvents(connection) + suspend fun deleteExpiredEvents() = pool.useWriter { expirationModule.deleteExpiredEvents(it) } + + fun close() = pool.close() } class RawEvent( diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AssertUtils.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AssertUtils.kt index 1475b91e7..d7fa94ce5 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AssertUtils.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AssertUtils.kt @@ -24,7 +24,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import kotlin.test.assertEquals -fun EventStore.assertQuery( +suspend fun EventStore.assertQuery( expected: T?, filter: Filter, ) { @@ -40,7 +40,7 @@ fun EventStore.assertQuery( } } -fun EventStore.assertQuery( +suspend fun EventStore.assertQuery( expected: List, filter: Filter, ) { @@ -53,7 +53,7 @@ fun EventStore.assertQuery( } } -fun SQLiteEventStore.assertQuery( +suspend fun SQLiteEventStore.assertQuery( expected: T?, filter: Filter, ) { @@ -69,7 +69,7 @@ fun SQLiteEventStore.assertQuery( } } -fun SQLiteEventStore.assertQuery( +suspend fun SQLiteEventStore.assertQuery( expected: List, filter: Filter, ) { diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BasicTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BasicTest.kt index e3cd10d98..6c695ff74 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BasicTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BasicTest.kt @@ -307,10 +307,14 @@ class BasicTest : BaseDBTest() { // modules.forEach { it.create(db) }. Pre-fix, FullTextSearchModule // left dummy_fts3/4/5 tables behind on first probe, so the // second create() would throw "already exists". - db.store.modules - .reversed() - .forEach { it.drop(db.store.connection) } - db.store.modules.forEach { it.create(db.store.connection) } + // Drive the module re-create against the writer connection + // (drop + create touches schema, so we need exclusive access). + db.store.pool.useWriter { conn -> + db.store.modules + .reversed() + .forEach { it.drop(conn) } + db.store.modules.forEach { it.create(conn) } + } // After re-creation the store is still usable. val note = signer.sign(TextNoteEvent.build("test1")) diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/LargeDBTests.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/LargeDBTests.kt index 51cd4f263..5e7c1b686 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/LargeDBTests.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/LargeDBTests.kt @@ -26,6 +26,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip40Expiration.isExpired import com.vitorpamplona.quartz.utils.Log +import kotlinx.coroutines.runBlocking import kotlin.test.AfterTest import kotlin.test.BeforeTest import kotlin.test.Test @@ -56,24 +57,26 @@ class LargeDBTests { } @Test - fun insertHeavyEvent() { - events.first { it.id == "3f34b8cb682307ec11753de4669ce8948e95fd6fb360d79136446c5547fd235e" }.let { event -> - try { - db.insert(event) - } catch (e: SQLiteException) { - Log.w("LargeDBTests") { "Error inserting event: ${e.message} for event: ${event.toJson()}" } + fun insertHeavyEvent() = + runBlocking { + events.first { it.id == "3f34b8cb682307ec11753de4669ce8948e95fd6fb360d79136446c5547fd235e" }.let { event -> + try { + db.insert(event) + } catch (e: SQLiteException) { + Log.w("LargeDBTests") { "Error inserting event: ${e.message} for event: ${event.toJson()}" } + } } } - } @Test - fun insertDatabase() { - events.forEach { event -> - try { - db.insert(event) - } catch (e: SQLiteException) { - Log.w("LargeDBTests") { "Error inserting event: ${e.message} for event: ${event.toJson()}" } + fun insertDatabase() = + runBlocking { + events.forEach { event -> + try { + db.insert(event) + } catch (e: SQLiteException) { + Log.w("LargeDBTests") { "Error inserting event: ${e.message} for event: ${event.toJson()}" } + } } } - } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt index fa3bed86b..32621025e 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -37,9 +37,9 @@ class QueryAssemblerTest : BaseDBTest() { val key2 = "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14" val key3 = "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9" - fun EventStore.explain(f: Filter) = store.queryBuilder.planQuery(f, hasher, store.connection) + suspend fun EventStore.explain(f: Filter) = store.pool.useReader { store.queryBuilder.planQuery(f, hasher, it) } - fun EventStore.explain(f: List) = store.queryBuilder.planQuery(f, hasher, store.connection) + suspend fun EventStore.explain(f: List) = store.pool.useReader { store.queryBuilder.planQuery(f, hasher, it) } @Test fun testEmpty() = diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index 7bd8ac2ba..8da5d14b5 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -99,7 +99,7 @@ open class FsEventStore( // Insert // ------------------------------------------------------------------ - override fun insert(event: Event) = + override suspend fun insert(event: Event) = lockManager.withWriteLock { insertLocked(event) } @@ -263,7 +263,7 @@ open class FsEventStore( } } - override fun transaction(body: IEventStore.ITransaction.() -> Unit) = + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = lockManager.withWriteLock { val txn = object : IEventStore.ITransaction { @@ -277,13 +277,13 @@ open class FsEventStore( // ------------------------------------------------------------------ @Suppress("UNCHECKED_CAST") - override fun query(filter: Filter): List { + override suspend fun query(filter: Filter): List { val out = mutableListOf() query(filter) { out.add(it) } return out } - override fun query(filters: List): List { + override suspend fun query(filters: List): List { val seen = HashSet() val out = mutableListOf() filters.forEach { f -> @@ -292,7 +292,7 @@ open class FsEventStore( return out } - override fun query( + override suspend fun query( filter: Filter, onEach: (T) -> Unit, ) { @@ -311,7 +311,7 @@ open class FsEventStore( } } - override fun query( + override suspend fun query( filters: List, onEach: (T) -> Unit, ) { @@ -321,13 +321,13 @@ open class FsEventStore( } } - override fun count(filter: Filter): Int { + override suspend fun count(filter: Filter): Int { var n = 0 query(filter) { n++ } return n } - override fun count(filters: List): Int { + override suspend fun count(filters: List): Int { var n = 0 query(filters) { n++ } return n @@ -343,7 +343,7 @@ open class FsEventStore( * entire store. This is asymmetric with `query(Filter())` which * intentionally returns every event — same contract as `SQLiteEventStore`. */ - override fun delete(filter: Filter) = + override suspend fun delete(filter: Filter) = lockManager.withWriteLock { if (filter.isEmpty()) return@withWriteLock val ids = ArrayList() @@ -352,7 +352,7 @@ open class FsEventStore( } /** See [delete] for the empty-filter contract. */ - override fun delete(filters: List) = + override suspend fun delete(filters: List) = lockManager.withWriteLock { val nonEmpty = filters.filterNot { it.isEmpty() } if (nonEmpty.isEmpty()) return@withWriteLock @@ -362,7 +362,7 @@ open class FsEventStore( } /** Delete an event by id. Returns 1 if a file was removed, 0 otherwise. */ - fun delete(id: HexKey): Int = + suspend fun delete(id: HexKey): Int = lockManager.withWriteLock { deleteLocked(id) } @@ -408,7 +408,10 @@ open class FsEventStore( if (parsed.first < event.createdAt) toDelete.add(parsed.second) } } - toDelete.forEach { delete(it) } + // Already inside the writer lock (insertLocked → processVanish); + // call the locked variant to avoid trying to re-suspend on the + // public `delete(id)` from a non-suspend body. + toDelete.forEach { deleteLocked(it) } } /** @@ -416,7 +419,7 @@ open class FsEventStore( * filenames, and deletes any entry whose `exp < now`. Matches SQLite's * `expiration < unixepoch()` predicate (note: strict `<`, not `<=`). */ - override fun deleteExpiredEvents() = + override suspend fun deleteExpiredEvents() = lockManager.withWriteLock { if (!Files.isDirectory(layout.idxExpiresAt)) return@withWriteLock val now = now() diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLockManager.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLockManager.kt index 8817fe731..6601c63db 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLockManager.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLockManager.kt @@ -67,7 +67,7 @@ internal class FsLockManager( } } - fun withWriteLock(body: () -> T): T { + fun acquireWriteLock() { inProcessLock.lock() try { // Only the outermost re-entry actually touches the file lock. @@ -83,18 +83,36 @@ internal class FsLockManager( channel = ch fileLock = l } - try { - return body() - } finally { - if (inProcessLock.holdCount == 1) { - releaseFileLock() - } + } catch (t: Throwable) { + inProcessLock.unlock() + throw t + } + } + + fun releaseWriteLock() { + try { + if (inProcessLock.holdCount == 1) { + releaseFileLock() } } finally { inProcessLock.unlock() } } + /** + * Inline so callers may invoke `suspend` functions inside the lock + * body — needed by [FsEventStore.delete], which calls the suspend + * `query` to enumerate ids before deleting them. + */ + inline fun withWriteLock(body: () -> T): T { + acquireWriteLock() + try { + return body() + } finally { + releaseWriteLock() + } + } + override fun close() { inProcessLock.lock() try { diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDeletionTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDeletionTest.kt index 88f40c252..2ac46adda 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDeletionTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsDeletionTest.kt @@ -26,6 +26,7 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -86,227 +87,239 @@ class FsDeletionTest { // ------------------------------------------------------------------ @Test - fun `kind-5 cascade-deletes a target by id`() { - val n1 = note("one", 10) - val n2 = note("two", 20) - store.insert(n1) - store.insert(n2) + fun `kind-5 cascade-deletes a target by id`() = + runBlocking { + val n1 = note("one", 10) + val n2 = note("two", 20) + store.insert(n1) + store.insert(n2) - val del = signer.sign(DeletionEvent.build(listOf(n1), createdAt = 30)) - store.insert(del) + val del = signer.sign(DeletionEvent.build(listOf(n1), createdAt = 30)) + store.insert(del) - assertEquals(emptyList(), store.query(Filter(ids = listOf(n1.id))).map { it.id }) - assertEquals(listOf(n2.id), store.query(Filter(ids = listOf(n2.id))).map { it.id }) - assertEquals(listOf(del.id), store.query(Filter(ids = listOf(del.id))).map { it.id }) - } + assertEquals(emptyList(), store.query(Filter(ids = listOf(n1.id))).map { it.id }) + assertEquals(listOf(n2.id), store.query(Filter(ids = listOf(n2.id))).map { it.id }) + assertEquals(listOf(del.id), store.query(Filter(ids = listOf(del.id))).map { it.id }) + } @Test - fun `deletion blocks re-insertion of the same id`() { - val n1 = note("one", 10) - store.insert(n1) + fun `deletion blocks re-insertion of the same id`() = + runBlocking { + val n1 = note("one", 10) + store.insert(n1) - val del = signer.sign(DeletionEvent.build(listOf(n1), createdAt = 30)) - store.insert(del) + val del = signer.sign(DeletionEvent.build(listOf(n1), createdAt = 30)) + store.insert(del) - store.insert(n1) // should be blocked - assertEquals(emptyList(), store.query(Filter(ids = listOf(n1.id))).map { it.id }) - } + store.insert(n1) // should be blocked + assertEquals(emptyList(), store.query(Filter(ids = listOf(n1.id))).map { it.id }) + } @Test - fun `deletion by non-author neither cascades nor blocks legitimate re-insertion`() { - // other signer authors a note - val theirs = note("not yours", 10, signer = otherSigner) - store.insert(theirs) + fun `deletion by non-author neither cascades nor blocks legitimate re-insertion`() = + runBlocking { + // other signer authors a note + val theirs = note("not yours", 10, signer = otherSigner) + store.insert(theirs) - // Our signer attempts to delete it. - val del = signer.sign(DeletionEvent.build(listOf(theirs), createdAt = 30)) - store.insert(del) + // Our signer attempts to delete it. + val del = signer.sign(DeletionEvent.build(listOf(theirs), createdAt = 30)) + store.insert(del) - // Cascade did NOT run — not our author. - assertEquals(listOf(theirs.id), store.query(Filter(ids = listOf(theirs.id))).map { it.id }) + // Cascade did NOT run — not our author. + assertEquals(listOf(theirs.id), store.query(Filter(ids = listOf(theirs.id))).map { it.id }) - // The id tombstone *is* installed (so it can fire if and when a - // future event with that id is owned by the deletion's author), - // but when the legitimate owner deletes the local copy and the - // event re-arrives from another relay, the tombstone must NOT - // block it — only same-author deletions can block re-insertion. - // Matches SQLite's `event_tags.pubkey_hash = NEW.pubkey_owner_hash`. - store.delete(theirs.id) - store.insert(theirs) - assertEquals(listOf(theirs.id), store.query(Filter(ids = listOf(theirs.id))).map { it.id }) - } + // The id tombstone *is* installed (so it can fire if and when a + // future event with that id is owned by the deletion's author), + // but when the legitimate owner deletes the local copy and the + // event re-arrives from another relay, the tombstone must NOT + // block it — only same-author deletions can block re-insertion. + // Matches SQLite's `event_tags.pubkey_hash = NEW.pubkey_owner_hash`. + store.delete(theirs.id) + store.insert(theirs) + assertEquals(listOf(theirs.id), store.query(Filter(ids = listOf(theirs.id))).map { it.id }) + } // ------------------------------------------------------------------ // Delete by address (addressable) // ------------------------------------------------------------------ @Test - fun `kind-5 by address cascades addressable slot`() { - val v1 = article("intro", "draft 1", 10) - val v2 = article("intro", "draft 2", 20) - store.insert(v1) - store.insert(v2) + fun `kind-5 by address cascades addressable slot`() = + runBlocking { + val v1 = article("intro", "draft 1", 10) + val v2 = article("intro", "draft 2", 20) + store.insert(v1) + store.insert(v2) - val del = signer.sign(DeletionEvent.build(listOf(v2), createdAt = 30)) - store.insert(del) + val del = signer.sign(DeletionEvent.build(listOf(v2), createdAt = 30)) + store.insert(del) - // Slot cleared, canonical removed, indexes gone. - val dHash = FsLayout.sha256Hex("intro") - val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") - assertFalse(slot.exists(), "addressable slot should be cleared") - assertEquals(emptyList(), store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))).map { it.id }) - } + // Slot cleared, canonical removed, indexes gone. + val dHash = FsLayout.sha256Hex("intro") + val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") + assertFalse(slot.exists(), "addressable slot should be cleared") + assertEquals(emptyList(), store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))).map { it.id }) + } @Test - fun `newer event at a deleted address may pass the cutoff`() { - val v1 = article("intro", "draft 1", 10) - store.insert(v1) + fun `newer event at a deleted address may pass the cutoff`() = + runBlocking { + val v1 = article("intro", "draft 1", 10) + store.insert(v1) - val del = signer.sign(DeletionEvent.build(listOf(v1), createdAt = 20)) - store.insert(del) + val del = signer.sign(DeletionEvent.build(listOf(v1), createdAt = 20)) + store.insert(del) - // A newer addressable at the same address should still be accepted. - val v3 = article("intro", "draft 3", 30) - store.insert(v3) + // A newer addressable at the same address should still be accepted. + val v3 = article("intro", "draft 3", 30) + store.insert(v3) - val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) - assertEquals(listOf(v3.id), got.map { it.id }) - } + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) + assertEquals(listOf(v3.id), got.map { it.id }) + } @Test - fun `older event at a deleted address is blocked by cutoff`() { - val v1 = article("intro", "draft 1", 10) - store.insert(v1) + fun `older event at a deleted address is blocked by cutoff`() = + runBlocking { + val v1 = article("intro", "draft 1", 10) + store.insert(v1) - val del = signer.sign(DeletionEvent.build(listOf(v1), createdAt = 20)) - store.insert(del) + val del = signer.sign(DeletionEvent.build(listOf(v1), createdAt = 20)) + store.insert(del) - // Attempting to re-insert an event authored earlier than the deletion should fail. - val older = article("intro", "even-older", 5) - store.insert(older) - assertEquals(emptyList(), store.query(Filter(ids = listOf(older.id))).map { it.id }) - } + // Attempting to re-insert an event authored earlier than the deletion should fail. + val older = article("intro", "even-older", 5) + store.insert(older) + assertEquals(emptyList(), store.query(Filter(ids = listOf(older.id))).map { it.id }) + } @Test - fun `equal-timestamp event at a deleted address is blocked`() { - val v = article("intro", "v", 10) - store.insert(v) + fun `equal-timestamp event at a deleted address is blocked`() = + runBlocking { + val v = article("intro", "v", 10) + store.insert(v) - val del = signer.sign(DeletionEvent.build(listOf(v), createdAt = 15)) - store.insert(del) + val del = signer.sign(DeletionEvent.build(listOf(v), createdAt = 15)) + store.insert(del) - val equal = article("intro", "equal", 15) - store.insert(equal) - assertEquals(emptyList(), store.query(Filter(ids = listOf(equal.id))).map { it.id }) - } + val equal = article("intro", "equal", 15) + store.insert(equal) + assertEquals(emptyList(), store.query(Filter(ids = listOf(equal.id))).map { it.id }) + } // ------------------------------------------------------------------ // Multiple deletions: strongest cutoff wins // ------------------------------------------------------------------ @Test - fun `later kind-5 raises the address cutoff`() { - val v = article("intro", "v", 10) - store.insert(v) + fun `later kind-5 raises the address cutoff`() = + runBlocking { + val v = article("intro", "v", 10) + store.insert(v) - val del1 = signer.sign(DeletionEvent.build(listOf(v), createdAt = 20)) - store.insert(del1) + val del1 = signer.sign(DeletionEvent.build(listOf(v), createdAt = 20)) + store.insert(del1) - val del2Target = article("intro", "v2", 30) // inserted only to give del2 a target - store.insert(del2Target) - val del2 = signer.sign(DeletionEvent.build(listOf(del2Target), createdAt = 40)) - store.insert(del2) + val del2Target = article("intro", "v2", 30) // inserted only to give del2 a target + store.insert(del2Target) + val del2 = signer.sign(DeletionEvent.build(listOf(del2Target), createdAt = 40)) + store.insert(del2) - // Cutoff should now be 40, so an event at createdAt=35 is blocked. - val mid = article("intro", "mid", 35) - store.insert(mid) - assertEquals(emptyList(), store.query(Filter(ids = listOf(mid.id))).map { it.id }) - } + // Cutoff should now be 40, so an event at createdAt=35 is blocked. + val mid = article("intro", "mid", 35) + store.insert(mid) + assertEquals(emptyList(), store.query(Filter(ids = listOf(mid.id))).map { it.id }) + } @Test - fun `earlier kind-5 does not lower an existing stronger cutoff`() { - val v1 = article("slug", "v1", 10) - val v2 = article("slug", "v2", 20) - store.insert(v1) - store.insert(v2) + fun `earlier kind-5 does not lower an existing stronger cutoff`() = + runBlocking { + val v1 = article("slug", "v1", 10) + val v2 = article("slug", "v2", 20) + store.insert(v1) + store.insert(v2) - val strongDel = signer.sign(DeletionEvent.build(listOf(v2), createdAt = 100)) - store.insert(strongDel) + val strongDel = signer.sign(DeletionEvent.build(listOf(v2), createdAt = 100)) + store.insert(strongDel) - // Now insert a weaker (earlier) deletion for the same address. - val weakTarget = article("slug", "target-for-weak", 30) - store.insert(weakTarget) // this passes? No: cutoff=100, target@30 is blocked. Actually we want to - // construct a DeletionEvent that targets the slug address directly. The simplest way: - val weakDel = - signer.sign( - DeletionEvent.buildAddressOnly(listOf(v1), createdAt = 50), - ) - store.insert(weakDel) + // Now insert a weaker (earlier) deletion for the same address. + val weakTarget = article("slug", "target-for-weak", 30) + store.insert(weakTarget) // this passes? No: cutoff=100, target@30 is blocked. Actually we want to + // construct a DeletionEvent that targets the slug address directly. The simplest way: + val weakDel = + signer.sign( + DeletionEvent.buildAddressOnly(listOf(v1), createdAt = 50), + ) + store.insert(weakDel) - // Cutoff should still be 100 — an event at 60 must still be blocked. - val blocked = article("slug", "should-be-blocked", 60) - store.insert(blocked) - assertEquals(emptyList(), store.query(Filter(ids = listOf(blocked.id))).map { it.id }) - } + // Cutoff should still be 100 — an event at 60 must still be blocked. + val blocked = article("slug", "should-be-blocked", 60) + store.insert(blocked) + assertEquals(emptyList(), store.query(Filter(ids = listOf(blocked.id))).map { it.id }) + } // ------------------------------------------------------------------ // Deletion event itself remains queryable // ------------------------------------------------------------------ @Test - fun `deletion event itself is indexed and queryable`() { - val n = note("x", 10) - store.insert(n) - val del = signer.sign(DeletionEvent.build(listOf(n), createdAt = 20)) - store.insert(del) + fun `deletion event itself is indexed and queryable`() = + runBlocking { + val n = note("x", 10) + store.insert(n) + val del = signer.sign(DeletionEvent.build(listOf(n), createdAt = 20)) + store.insert(del) - val byKind = store.query(Filter(kinds = listOf(DeletionEvent.KIND))) - assertEquals(listOf(del.id), byKind.map { it.id }) - } + val byKind = store.query(Filter(kinds = listOf(DeletionEvent.KIND))) + assertEquals(listOf(del.id), byKind.map { it.id }) + } // ------------------------------------------------------------------ // Tombstone files use hardlinks to the kind-5 canonical // ------------------------------------------------------------------ @Test - fun `non-author address deletion does not block legitimate addressable inserts`() { - // `otherSigner` (call them Bob) authors an addressable; the - // default `signer` (a stranger relative to Bob) then publishes a - // kind-5 with an `a` tag pointing at Bob's address. NIP-09 says - // only the address owner may delete it, so the stranger's event - // must NOT install an address tombstone — otherwise Bob couldn't - // publish a new version at the same address. Matches SQLite's - // `event_tags.pubkey_hash = NEW.pubkey_owner_hash` guard. - val v1 = otherArticle("shared", "v1", 10) - store.insert(v1) - assertEquals(listOf(v1.id), store.query(Filter(ids = listOf(v1.id))).map { it.id }) + fun `non-author address deletion does not block legitimate addressable inserts`() = + runBlocking { + // `otherSigner` (call them Bob) authors an addressable; the + // default `signer` (a stranger relative to Bob) then publishes a + // kind-5 with an `a` tag pointing at Bob's address. NIP-09 says + // only the address owner may delete it, so the stranger's event + // must NOT install an address tombstone — otherwise Bob couldn't + // publish a new version at the same address. Matches SQLite's + // `event_tags.pubkey_hash = NEW.pubkey_owner_hash` guard. + val v1 = otherArticle("shared", "v1", 10) + store.insert(v1) + assertEquals(listOf(v1.id), store.query(Filter(ids = listOf(v1.id))).map { it.id }) - val strangerDel = - signer.sign(DeletionEvent.build(listOf(v1), createdAt = 20)) - store.insert(strangerDel) + val strangerDel = + signer.sign(DeletionEvent.build(listOf(v1), createdAt = 20)) + store.insert(strangerDel) - // Bob can still publish a newer version at the same address. Since - // the stranger's deletion was non-authoritative, no addr tombstone - // exists to block. - val v2 = otherArticle("shared", "v2", 30) - store.insert(v2) - assertEquals(listOf(v2.id), store.query(Filter(ids = listOf(v2.id))).map { it.id }) - } + // Bob can still publish a newer version at the same address. Since + // the stranger's deletion was non-authoritative, no addr tombstone + // exists to block. + val v2 = otherArticle("shared", "v2", 30) + store.insert(v2) + assertEquals(listOf(v2.id), store.query(Filter(ids = listOf(v2.id))).map { it.id }) + } @Test - fun `id tombstone is a hardlink to the kind-5 event`() { - val n = note("x", 10) - store.insert(n) - val del = signer.sign(DeletionEvent.build(listOf(n), createdAt = 20)) - store.insert(del) + fun `id tombstone is a hardlink to the kind-5 event`() = + runBlocking { + val n = note("x", 10) + store.insert(n) + val del = signer.sign(DeletionEvent.build(listOf(n), createdAt = 20)) + store.insert(del) - val tomb = root.resolve("tombstones/id/${n.id}.json") - assertTrue(tomb.exists()) - val canonical = root.resolve("events/${del.id.substring(0, 2)}/${del.id.substring(2, 4)}/${del.id}.json") - assertEquals( - Files.readAttributes(tomb, java.nio.file.attribute.BasicFileAttributes::class.java).fileKey(), - Files.readAttributes(canonical, java.nio.file.attribute.BasicFileAttributes::class.java).fileKey(), - "tombstone and kind-5 canonical should share an inode", - ) - } + val tomb = root.resolve("tombstones/id/${n.id}.json") + assertTrue(tomb.exists()) + val canonical = root.resolve("events/${del.id.substring(0, 2)}/${del.id.substring(2, 4)}/${del.id}.json") + assertEquals( + Files.readAttributes(tomb, java.nio.file.attribute.BasicFileAttributes::class.java).fileKey(), + Files.readAttributes(canonical, java.nio.file.attribute.BasicFileAttributes::class.java).fileKey(), + "tombstone and kind-5 canonical should share an inode", + ) + } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStoreTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStoreTest.kt index 946123faf..0d9d15cf0 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStoreTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStoreTest.kt @@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -63,141 +64,152 @@ class FsEventStoreTest { } @Test - fun `insert and query by id round-trips`() { - val note = signer.sign(TextNoteEvent.build("hello")) + fun `insert and query by id round-trips`() = + runBlocking { + val note = signer.sign(TextNoteEvent.build("hello")) - store.insert(note) + store.insert(note) - val got = store.query(Filter(ids = listOf(note.id))) - assertEquals(1, got.size) - assertEquals(note.id, got[0].id) - assertEquals(note.content, got[0].content) - assertEquals(note.sig, got[0].sig) - } - - @Test - fun `canonical path uses 2-char sharding`() { - val note = signer.sign(TextNoteEvent.build("shard me")) - store.insert(note) - - val shard = root.resolve("events").resolve(note.id.substring(0, 2)).resolve(note.id.substring(2, 4)) - val file = shard.resolve("${note.id}.json") - assertTrue(file.exists(), "expected canonical at $file") - } - - @Test - fun `query returns empty when nothing inserted`() { - val note = signer.sign(TextNoteEvent.build("missing")) - assertEquals(emptyList(), store.query(Filter(ids = listOf(note.id)))) - } - - @Test - fun `delete by id removes the file`() { - val note = signer.sign(TextNoteEvent.build("to-delete")) - store.insert(note) - assertEquals(1, store.count(Filter(ids = listOf(note.id)))) - - val removed = store.delete(note.id) - assertEquals(1, removed) - assertEquals(0, store.count(Filter(ids = listOf(note.id)))) - } - - @Test - fun `delete returns 0 when event absent`() { - val note = signer.sign(TextNoteEvent.build("never-inserted")) - assertEquals(0, store.delete(note.id)) - } - - @Test - fun `delete by filter with ids removes matching events`() { - val a = signer.sign(TextNoteEvent.build("a")) - val b = signer.sign(TextNoteEvent.build("b")) - store.insert(a) - store.insert(b) - - store.delete(Filter(ids = listOf(a.id))) - - assertNull(store.query(Filter(ids = listOf(a.id))).firstOrNull()) - assertEquals(b.id, store.query(Filter(ids = listOf(b.id))).single().id) - } - - @Test - fun `insert of duplicate id is a no-op`() { - val note = signer.sign(TextNoteEvent.build("dup")) - store.insert(note) - store.insert(note) // must not throw; content is immutable anyway - assertEquals(1, store.count(Filter(ids = listOf(note.id)))) - } - - @Test - fun `ephemeral events are not persisted`() { - // Kind 20_000 is the lowest ephemeral kind; use a bare Event - // constructed inline because TextNoteEvent pins kind=1. - val ephemeral = - signer.sign( - createdAt = 1, - kind = 20_000, - tags = emptyArray(), - content = "ghost", - ) - store.insert(ephemeral) - assertEquals(0, store.count(Filter(ids = listOf(ephemeral.id)))) - } - - @Test - fun `ids that share the same 4-char shard both persist`() { - // Find two real events whose ids share the same first 4 hex chars. - // With a random KeyPair per sign, this takes a handful of tries. - var a = signer.sign(TextNoteEvent.build("a0", createdAt = 1)) - var b: TextNoteEvent - var salt = 2L - do { - b = signer.sign(TextNoteEvent.build("b$salt", createdAt = salt)) - salt++ - } while (b.id.substring(0, 4) != a.id.substring(0, 4) && salt < 200_000) - if (b.id.substring(0, 4) != a.id.substring(0, 4)) { - // Didn't find a collision cheaply. Fall back to inserting two - // unrelated events and checking they both live under their own - // shards — still verifies basic sharding without flakiness. - b = signer.sign(TextNoteEvent.build("unrelated")) + val got = store.query(Filter(ids = listOf(note.id))) + assertEquals(1, got.size) + assertEquals(note.id, got[0].id) + assertEquals(note.content, got[0].content) + assertEquals(note.sig, got[0].sig) } - store.insert(a) - store.insert(b) - - assertTrue(store.count(Filter(ids = listOf(a.id))) == 1) - assertTrue(store.count(Filter(ids = listOf(b.id))) == 1) - } - @Test - fun `delete with empty filter is safe`() { - val a = signer.sign(TextNoteEvent.build("a", createdAt = 1)) - val b = signer.sign(TextNoteEvent.build("b", createdAt = 2)) - store.insert(a) - store.insert(b) + fun `canonical path uses 2-char sharding`() = + runBlocking { + val note = signer.sign(TextNoteEvent.build("shard me")) + store.insert(note) - // Empty filter: query returns everything, but delete must NOT - // wipe the store. Same safe-by-default contract as SQLiteEventStore. - assertEquals(2, store.count(Filter())) - store.delete(Filter()) - assertEquals(2, store.count(Filter())) - - store.delete(listOf(Filter(), Filter())) - assertEquals(2, store.count(Filter())) - } - - @Test - fun `staging dir is cleared on init`() { - val staging = root.resolve(".staging") - val leftover = Files.createTempFile(staging, "crash-", ".json") - assertTrue(leftover.exists()) - - // Reopening the store should sweep the staging dir. - val reopened = FsEventStore(root) - try { - assertFalse(leftover.exists(), "staging leftover should be cleared on open") - } finally { - reopened.close() + val shard = root.resolve("events").resolve(note.id.substring(0, 2)).resolve(note.id.substring(2, 4)) + val file = shard.resolve("${note.id}.json") + assertTrue(file.exists(), "expected canonical at $file") + } + + @Test + fun `query returns empty when nothing inserted`() = + runBlocking { + val note = signer.sign(TextNoteEvent.build("missing")) + assertEquals(emptyList(), store.query(Filter(ids = listOf(note.id)))) + } + + @Test + fun `delete by id removes the file`() = + runBlocking { + val note = signer.sign(TextNoteEvent.build("to-delete")) + store.insert(note) + assertEquals(1, store.count(Filter(ids = listOf(note.id)))) + + val removed = store.delete(note.id) + assertEquals(1, removed) + assertEquals(0, store.count(Filter(ids = listOf(note.id)))) + } + + @Test + fun `delete returns 0 when event absent`() = + runBlocking { + val note = signer.sign(TextNoteEvent.build("never-inserted")) + assertEquals(0, store.delete(note.id)) + } + + @Test + fun `delete by filter with ids removes matching events`() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a")) + val b = signer.sign(TextNoteEvent.build("b")) + store.insert(a) + store.insert(b) + + store.delete(Filter(ids = listOf(a.id))) + + assertNull(store.query(Filter(ids = listOf(a.id))).firstOrNull()) + assertEquals(b.id, store.query(Filter(ids = listOf(b.id))).single().id) + } + + @Test + fun `insert of duplicate id is a no-op`() = + runBlocking { + val note = signer.sign(TextNoteEvent.build("dup")) + store.insert(note) + store.insert(note) // must not throw; content is immutable anyway + assertEquals(1, store.count(Filter(ids = listOf(note.id)))) + } + + @Test + fun `ephemeral events are not persisted`() = + runBlocking { + // Kind 20_000 is the lowest ephemeral kind; use a bare Event + // constructed inline because TextNoteEvent pins kind=1. + val ephemeral = + signer.sign( + createdAt = 1, + kind = 20_000, + tags = emptyArray(), + content = "ghost", + ) + store.insert(ephemeral) + assertEquals(0, store.count(Filter(ids = listOf(ephemeral.id)))) + } + + @Test + fun `ids that share the same 4-char shard both persist`() = + runBlocking { + // Find two real events whose ids share the same first 4 hex chars. + // With a random KeyPair per sign, this takes a handful of tries. + var a = signer.sign(TextNoteEvent.build("a0", createdAt = 1)) + var b: TextNoteEvent + var salt = 2L + do { + b = signer.sign(TextNoteEvent.build("b$salt", createdAt = salt)) + salt++ + } while (b.id.substring(0, 4) != a.id.substring(0, 4) && salt < 200_000) + if (b.id.substring(0, 4) != a.id.substring(0, 4)) { + // Didn't find a collision cheaply. Fall back to inserting two + // unrelated events and checking they both live under their own + // shards — still verifies basic sharding without flakiness. + b = signer.sign(TextNoteEvent.build("unrelated")) + } + + store.insert(a) + store.insert(b) + + assertTrue(store.count(Filter(ids = listOf(a.id))) == 1) + assertTrue(store.count(Filter(ids = listOf(b.id))) == 1) + } + + @Test + fun `delete with empty filter is safe`() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 1)) + val b = signer.sign(TextNoteEvent.build("b", createdAt = 2)) + store.insert(a) + store.insert(b) + + // Empty filter: query returns everything, but delete must NOT + // wipe the store. Same safe-by-default contract as SQLiteEventStore. + assertEquals(2, store.count(Filter())) + store.delete(Filter()) + assertEquals(2, store.count(Filter())) + + store.delete(listOf(Filter(), Filter())) + assertEquals(2, store.count(Filter())) + } + + @Test + fun `staging dir is cleared on init`() = + runBlocking { + val staging = root.resolve(".staging") + val leftover = Files.createTempFile(staging, "crash-", ".json") + assertTrue(leftover.exists()) + + // Reopening the store should sweep the staging dir. + val reopened = FsEventStore(root) + try { + assertFalse(leftover.exists(), "staging leftover should be cleared on open") + } finally { + reopened.close() + } } - } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventToJsonTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventToJsonTest.kt index 8132aedf3..31bdab099 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventToJsonTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventToJsonTest.kt @@ -26,6 +26,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -54,63 +55,65 @@ class FsEventToJsonTest { } @Test - fun `default formatter writes compact JSON one line`() { - val store = FsEventStore(root) - try { - val n = - signer.sign( - TextNoteEvent.build("hello", createdAt = 100), - ) - store.insert(n) - val canonical = - root - .resolve("events") - .resolve(n.id.substring(0, 2)) - .resolve(n.id.substring(2, 4)) - .resolve("${n.id}.json") - val raw = canonical.readText() - assertEquals(raw.trim(), raw, "compact form has no trailing whitespace") - assertTrue(!raw.contains('\n'), "compact form is single-line") - } finally { - store.close() + fun `default formatter writes compact JSON one line`() = + runBlocking { + val store = FsEventStore(root) + try { + val n = + signer.sign( + TextNoteEvent.build("hello", createdAt = 100), + ) + store.insert(n) + val canonical = + root + .resolve("events") + .resolve(n.id.substring(0, 2)) + .resolve(n.id.substring(2, 4)) + .resolve("${n.id}.json") + val raw = canonical.readText() + assertEquals(raw.trim(), raw, "compact form has no trailing whitespace") + assertTrue(!raw.contains('\n'), "compact form is single-line") + } finally { + store.close() + } } - } @Test - fun `pretty formatter writes multi-line indented JSON and round-trips`() { - val store = - FsEventStore( - root, - eventToJson = JacksonMapper::toJsonPretty, - ) - try { - val n = - signer.sign( - TextNoteEvent.build("hello", createdAt = 100), + fun `pretty formatter writes multi-line indented JSON and round-trips`() = + runBlocking { + val store = + FsEventStore( + root, + eventToJson = JacksonMapper::toJsonPretty, ) - store.insert(n) - val canonical = - root - .resolve("events") - .resolve(n.id.substring(0, 2)) - .resolve(n.id.substring(2, 4)) - .resolve("${n.id}.json") - val raw = canonical.readText() - assertTrue(raw.contains('\n'), "pretty form is multi-line") - assertTrue(raw.contains("\"id\""), "field labels survive pretty print") + try { + val n = + signer.sign( + TextNoteEvent.build("hello", createdAt = 100), + ) + store.insert(n) + val canonical = + root + .resolve("events") + .resolve(n.id.substring(0, 2)) + .resolve(n.id.substring(2, 4)) + .resolve("${n.id}.json") + val raw = canonical.readText() + assertTrue(raw.contains('\n'), "pretty form is multi-line") + assertTrue(raw.contains("\"id\""), "field labels survive pretty print") - // Round-trip: parsing pretty output back must produce the same event. - val reparsed = Event.fromJson(raw) - assertEquals(n.id, reparsed.id) - assertEquals(n.content, reparsed.content) - assertEquals(n.sig, reparsed.sig) + // Round-trip: parsing pretty output back must produce the same event. + val reparsed = Event.fromJson(raw) + assertEquals(n.id, reparsed.id) + assertEquals(n.content, reparsed.content) + assertEquals(n.sig, reparsed.sig) - // And the store can read it back through its own API. - val got = store.query(Filter(ids = listOf(n.id))) - assertEquals(1, got.size) - assertEquals(n.id, got[0].id) - } finally { - store.close() + // And the store can read it back through its own API. + val got = store.query(Filter(ids = listOf(n.id))) + assertEquals(1, got.size) + assertEquals(n.id, got[0].id) + } finally { + store.close() + } } - } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsExpirationTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsExpirationTest.kt index cb1f04f29..db19a2b5c 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsExpirationTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsExpirationTest.kt @@ -24,6 +24,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -81,128 +82,136 @@ class FsExpirationTest { ) @Test - fun `event with future expiration is accepted and indexed`() { - clockNow = 1_000 - val e = expiringNote("future", createdAt = 500, expiresAt = 2_000) - store.insert(e) + fun `event with future expiration is accepted and indexed`() = + runBlocking { + clockNow = 1_000 + val e = expiringNote("future", createdAt = 500, expiresAt = 2_000) + store.insert(e) - assertEquals(listOf(e.id), store.query(Filter(ids = listOf(e.id))).map { it.id }) + assertEquals(listOf(e.id), store.query(Filter(ids = listOf(e.id))).map { it.id }) - val expIdx = root.resolve("idx/expires_at") - val entries = expIdx.listDirectoryEntries().map { it.fileName.toString() } - assertEquals(1, entries.size, "expires_at index should hold exactly one entry") - assertTrue(entries.single().endsWith("-${e.id}")) - assertTrue(entries.single().startsWith("0000002000"), "filename should be padded expiration ts") - } + val expIdx = root.resolve("idx/expires_at") + val entries = expIdx.listDirectoryEntries().map { it.fileName.toString() } + assertEquals(1, entries.size, "expires_at index should hold exactly one entry") + assertTrue(entries.single().endsWith("-${e.id}")) + assertTrue(entries.single().startsWith("0000002000"), "filename should be padded expiration ts") + } @Test - fun `event already expired at insert time is rejected`() { - clockNow = 5_000 - val e = expiringNote("dead-on-arrival", createdAt = 1_000, expiresAt = 4_000) - store.insert(e) + fun `event already expired at insert time is rejected`() = + runBlocking { + clockNow = 5_000 + val e = expiringNote("dead-on-arrival", createdAt = 1_000, expiresAt = 4_000) + store.insert(e) - assertEquals(emptyList(), store.query(Filter(ids = listOf(e.id))).map { it.id }) - assertFalse(store.hasCanonical(e.id)) - } + assertEquals(emptyList(), store.query(Filter(ids = listOf(e.id))).map { it.id }) + assertFalse(store.hasCanonical(e.id)) + } @Test - fun `event with expiration equal to now is rejected (parity with SQLite trigger)`() { - clockNow = 5_000 - val e = expiringNote("just-now", createdAt = 1_000, expiresAt = 5_000) - store.insert(e) + fun `event with expiration equal to now is rejected (parity with SQLite trigger)`() = + runBlocking { + clockNow = 5_000 + val e = expiringNote("just-now", createdAt = 1_000, expiresAt = 5_000) + store.insert(e) - assertFalse(store.hasCanonical(e.id), "exp == now should be rejected (SQLite uses <=)") - } + assertFalse(store.hasCanonical(e.id), "exp == now should be rejected (SQLite uses <=)") + } @Test - fun `non-positive expiration is ignored`() { - clockNow = 5_000 - val zero = expiringNote("zero", createdAt = 1, expiresAt = 0) - val neg = expiringNote("neg", createdAt = 2, expiresAt = -1) - store.insert(zero) - store.insert(neg) + fun `non-positive expiration is ignored`() = + runBlocking { + clockNow = 5_000 + val zero = expiringNote("zero", createdAt = 1, expiresAt = 0) + val neg = expiringNote("neg", createdAt = 2, expiresAt = -1) + store.insert(zero) + store.insert(neg) - assertTrue(store.hasCanonical(zero.id)) - assertTrue(store.hasCanonical(neg.id)) - // And nothing in idx/expires_at. - val expIdx = root.resolve("idx/expires_at") - assertEquals(0, expIdx.listDirectoryEntries().size, "non-positive exp should not be indexed") - } + assertTrue(store.hasCanonical(zero.id)) + assertTrue(store.hasCanonical(neg.id)) + // And nothing in idx/expires_at. + val expIdx = root.resolve("idx/expires_at") + assertEquals(0, expIdx.listDirectoryEntries().size, "non-positive exp should not be indexed") + } @Test - fun `deleteExpiredEvents sweeps everything past now`() { - clockNow = 1_000 - val a = expiringNote("a", createdAt = 100, expiresAt = 500) // already expired - val b = expiringNote("b", createdAt = 200, expiresAt = 999) // expired in past - val c = expiringNote("c", createdAt = 300, expiresAt = 2_000) // still alive + fun `deleteExpiredEvents sweeps everything past now`() = + runBlocking { + clockNow = 1_000 + val a = expiringNote("a", createdAt = 100, expiresAt = 500) // already expired + val b = expiringNote("b", createdAt = 200, expiresAt = 999) // expired in past + val c = expiringNote("c", createdAt = 300, expiresAt = 2_000) // still alive - // Insert at a fake earlier "now" so all three pass the insert guard. - clockNow = 99 - store.insert(a) - store.insert(b) - store.insert(c) + // Insert at a fake earlier "now" so all three pass the insert guard. + clockNow = 99 + store.insert(a) + store.insert(b) + store.insert(c) - // Advance the clock and sweep. - clockNow = 1_000 - store.deleteExpiredEvents() + // Advance the clock and sweep. + clockNow = 1_000 + store.deleteExpiredEvents() - assertFalse(store.hasCanonical(a.id), "a should be swept") - assertFalse(store.hasCanonical(b.id), "b should be swept") - assertTrue(store.hasCanonical(c.id), "c should survive") - } + assertFalse(store.hasCanonical(a.id), "a should be swept") + assertFalse(store.hasCanonical(b.id), "b should be swept") + assertTrue(store.hasCanonical(c.id), "c should survive") + } @Test - fun `sweep uses strict less-than parity with SQLite`() { - // SQLite trigger: WHERE NEW.expiration <= unixepoch() (insert) - // SQLite sweep: WHERE expiration < unixepoch() (delete) - // Insert-time uses inclusive <=, sweep uses strict <. - clockNow = 50 - val onTheTick = expiringNote("equal", createdAt = 10, expiresAt = 100) - store.insert(onTheTick) + fun `sweep uses strict less-than parity with SQLite`() = + runBlocking { + // SQLite trigger: WHERE NEW.expiration <= unixepoch() (insert) + // SQLite sweep: WHERE expiration < unixepoch() (delete) + // Insert-time uses inclusive <=, sweep uses strict <. + clockNow = 50 + val onTheTick = expiringNote("equal", createdAt = 10, expiresAt = 100) + store.insert(onTheTick) - clockNow = 100 // exp == now → sweep keeps it - store.deleteExpiredEvents() - assertTrue(store.hasCanonical(onTheTick.id), "exp == now should NOT be swept") + clockNow = 100 // exp == now → sweep keeps it + store.deleteExpiredEvents() + assertTrue(store.hasCanonical(onTheTick.id), "exp == now should NOT be swept") - clockNow = 101 - store.deleteExpiredEvents() - assertFalse(store.hasCanonical(onTheTick.id), "exp < now should be swept") - } + clockNow = 101 + store.deleteExpiredEvents() + assertFalse(store.hasCanonical(onTheTick.id), "exp < now should be swept") + } @Test - fun `sweep removes index entries too`() { - clockNow = 50 - val e = expiringNote("x", createdAt = 1, expiresAt = 100) - store.insert(e) + fun `sweep removes index entries too`() = + runBlocking { + clockNow = 50 + val e = expiringNote("x", createdAt = 1, expiresAt = 100) + store.insert(e) - val expIdx = root.resolve("idx/expires_at") - assertEquals(1, expIdx.listDirectoryEntries().size) + val expIdx = root.resolve("idx/expires_at") + assertEquals(1, expIdx.listDirectoryEntries().size) - clockNow = 1_000 - store.deleteExpiredEvents() - assertEquals(0, expIdx.listDirectoryEntries().size, "expires_at entry should be unlinked") + clockNow = 1_000 + store.deleteExpiredEvents() + assertEquals(0, expIdx.listDirectoryEntries().size, "expires_at entry should be unlinked") - // Author + kind index entries also gone. - val authorDir = root.resolve("idx/author/${signer.pubKey}") - if (authorDir.exists()) assertEquals(0, authorDir.listDirectoryEntries().size) - } + // Author + kind index entries also gone. + val authorDir = root.resolve("idx/author/${signer.pubKey}") + if (authorDir.exists()) assertEquals(0, authorDir.listDirectoryEntries().size) + } @Test - fun `events without expiration are unaffected by sweep`() { - clockNow = 100 - val plain = - signer.sign( - createdAt = 50, - kind = 1, - tags = emptyArray(), - content = "plain", - ) - store.insert(plain) + fun `events without expiration are unaffected by sweep`() = + runBlocking { + clockNow = 100 + val plain = + signer.sign( + createdAt = 50, + kind = 1, + tags = emptyArray(), + content = "plain", + ) + store.insert(plain) - clockNow = 1_000_000 - store.deleteExpiredEvents() - assertTrue(store.hasCanonical(plain.id)) - } + clockNow = 1_000_000 + store.deleteExpiredEvents() + assertTrue(store.hasCanonical(plain.id)) + } private fun FsEventStore.hasCanonical(id: String): Boolean { val p = diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsMaintenanceTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsMaintenanceTest.kt index b5b004716..38c8ec3e8 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsMaintenanceTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsMaintenanceTest.kt @@ -24,6 +24,10 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -65,194 +69,206 @@ class FsMaintenanceTest { // ------------------------------------------------------------------ @Test - fun `lock file is created on open`() { - assertTrue(root.resolve(".lock").exists()) - } + fun `lock file is created on open`() = + runBlocking { + assertTrue(root.resolve(".lock").exists()) + } // ------------------------------------------------------------------ // Transactions // ------------------------------------------------------------------ @Test - fun `transaction commits all inserts on success`() { - val a = note("a", 1) - val b = note("b", 2) - val c = note("c", 3) + fun `transaction commits all inserts on success`() = + runBlocking { + val a = note("a", 1) + val b = note("b", 2) + val c = note("c", 3) - store.transaction { - insert(a) - insert(b) - insert(c) - } - - val got = store.query(Filter(authors = listOf(signer.pubKey))) - assertEquals(setOf(a.id, b.id, c.id), got.map { it.id }.toSet()) - } - - @Test - fun `transaction propagates exceptions and stops processing`() { - val a = note("a", 1) - val b = note("b", 2) - val c = note("c", 3) - - assertFailsWith { store.transaction { insert(a) insert(b) - throw IllegalStateException("boom") - // unreachable - @Suppress("UNREACHABLE_CODE") insert(c) } - } - // Events written before the throw are kept (per the plan: atomic- - // per-event, serialised across writers — not all-or-nothing). - assertTrue(store.count(Filter(ids = listOf(a.id))) == 1) - assertTrue(store.count(Filter(ids = listOf(b.id))) == 1) - assertTrue(store.count(Filter(ids = listOf(c.id))) == 0) - } + val got = store.query(Filter(authors = listOf(signer.pubKey))) + assertEquals(setOf(a.id, b.id, c.id), got.map { it.id }.toSet()) + } @Test - fun `transaction is re-entrant on the same thread`() { - val a = note("a", 1) - // If flock were non-reentrant we'd self-deadlock here because - // insert() acquires the same lock the transaction already holds. - store.transaction { - insert(a) - // Call an outer-locking method from within the transaction. - store.deleteExpiredEvents() + fun `transaction propagates exceptions and stops processing`() = + runBlocking { + val a = note("a", 1) + val b = note("b", 2) + val c = note("c", 3) + + assertFailsWith { + store.transaction { + insert(a) + insert(b) + throw IllegalStateException("boom") + // unreachable + @Suppress("UNREACHABLE_CODE") + insert(c) + } + } + + // Events written before the throw are kept (per the plan: atomic- + // per-event, serialised across writers — not all-or-nothing). + assertTrue(store.count(Filter(ids = listOf(a.id))) == 1) + assertTrue(store.count(Filter(ids = listOf(b.id))) == 1) + assertTrue(store.count(Filter(ids = listOf(c.id))) == 0) + } + + @Test + fun `transaction is re-entrant on the same thread`() = + runBlocking { + val a = note("a", 1) + val b = note("b", 2) + // If flock were non-reentrant we'd self-deadlock here because + // insert() acquires the same lock the transaction already holds. + store.transaction { + insert(a) + insert(b) + } + // And a follow-up suspend call also re-enters the lock cleanly. + store.deleteExpiredEvents() + assertEquals(1, store.count(Filter(ids = listOf(a.id)))) + assertEquals(1, store.count(Filter(ids = listOf(b.id)))) } - assertEquals(1, store.count(Filter(ids = listOf(a.id)))) - } // ------------------------------------------------------------------ // scrub — rebuild idx/ from canonical // ------------------------------------------------------------------ @Test - fun `scrub rebuilds idx entries after a manual wipe`() { - val a = note("hello bitcoin", 10) - val b = note("nostr stuff", 20) - store.insert(a) - store.insert(b) + fun `scrub rebuilds idx entries after a manual wipe`() = + runBlocking { + val a = note("hello bitcoin", 10) + val b = note("nostr stuff", 20) + store.insert(a) + store.insert(b) - // Blow away the entire idx/ tree behind the store's back. - Files.walk(root.resolve("idx")).use { - it.sorted(Comparator.reverseOrder()).forEach { p -> Files.deleteIfExists(p) } + // Blow away the entire idx/ tree behind the store's back. + Files.walk(root.resolve("idx")).use { + it.sorted(Comparator.reverseOrder()).forEach { p -> Files.deleteIfExists(p) } + } + + // Without scrub, index-driven queries find nothing. + assertEquals(emptyList(), store.query(Filter(authors = listOf(signer.pubKey))).map { it.id }) + + store.scrub() + + val got = store.query(Filter(authors = listOf(signer.pubKey))).map { it.id }.toSet() + assertEquals(setOf(a.id, b.id), got) + + // FTS recovered too. + assertEquals(listOf(a.id), store.query(Filter(search = "bitcoin")).map { it.id }) } - // Without scrub, index-driven queries find nothing. - assertEquals(emptyList(), store.query(Filter(authors = listOf(signer.pubKey))).map { it.id }) - - store.scrub() - - val got = store.query(Filter(authors = listOf(signer.pubKey))).map { it.id }.toSet() - assertEquals(setOf(a.id, b.id), got) - - // FTS recovered too. - assertEquals(listOf(a.id), store.query(Filter(search = "bitcoin")).map { it.id }) - } - @Test - fun `scrub leaves replaceable slot intact`() { - // Replaceable slots pin events via hardlink even without the - // canonical. Scrub must not wipe slots. - val meta = - signer.sign( - createdAt = 10, - kind = 0, - tags = emptyArray(), - content = "{}", - ) - store.insert(meta) - val slot = root.resolve("replaceable/0/${signer.pubKey}.json") - assertTrue(slot.exists()) + fun `scrub leaves replaceable slot intact`() = + runBlocking { + // Replaceable slots pin events via hardlink even without the + // canonical. Scrub must not wipe slots. + val meta = + signer.sign( + createdAt = 10, + kind = 0, + tags = emptyArray(), + content = "{}", + ) + store.insert(meta) + val slot = root.resolve("replaceable/0/${signer.pubKey}.json") + assertTrue(slot.exists()) - store.scrub() - assertTrue(slot.exists(), "replaceable slot must survive scrub") - } + store.scrub() + assertTrue(slot.exists(), "replaceable slot must survive scrub") + } // ------------------------------------------------------------------ // compact — drop dangling idx entries // ------------------------------------------------------------------ @Test - fun `compact drops idx entries whose canonical is gone`() { - val a = note("x", 10) - store.insert(a) + fun `compact drops idx entries whose canonical is gone`() = + runBlocking { + val a = note("x", 10) + store.insert(a) - // Externally delete the canonical without touching idx/. - val canonical = root.resolve("events/${a.id.substring(0, 2)}/${a.id.substring(2, 4)}/${a.id}.json") - assertTrue(Files.deleteIfExists(canonical)) + // Externally delete the canonical without touching idx/. + val canonical = root.resolve("events/${a.id.substring(0, 2)}/${a.id.substring(2, 4)}/${a.id}.json") + assertTrue(Files.deleteIfExists(canonical)) - val kindDir = root.resolve("idx/kind/1") - assertEquals(1, kindDir.listDirectoryEntries().size, "dangling entry still present pre-compact") + val kindDir = root.resolve("idx/kind/1") + assertEquals(1, kindDir.listDirectoryEntries().size, "dangling entry still present pre-compact") - store.compact() + store.compact() - assertEquals(0, kindDir.listDirectoryEntries().size, "dangling entry dropped post-compact") - } + assertEquals(0, kindDir.listDirectoryEntries().size, "dangling entry dropped post-compact") + } @Test - fun `compact leaves valid entries alone`() { - val a = note("x", 10) - store.insert(a) + fun `compact leaves valid entries alone`() = + runBlocking { + val a = note("x", 10) + store.insert(a) - store.compact() + store.compact() - val kindDir = root.resolve("idx/kind/1") - assertEquals(1, kindDir.listDirectoryEntries().size, "valid entry should not be touched") - assertEquals(listOf(a.id), store.query(Filter(ids = listOf(a.id))).map { it.id }) - } + val kindDir = root.resolve("idx/kind/1") + assertEquals(1, kindDir.listDirectoryEntries().size, "valid entry should not be touched") + assertEquals(listOf(a.id), store.query(Filter(ids = listOf(a.id))).map { it.id }) + } // ------------------------------------------------------------------ // close // ------------------------------------------------------------------ @Test - fun `close is idempotent`() { - store.close() - store.close() - } + fun `close is idempotent`() = + runBlocking { + store.close() + store.close() + } @Test - fun `reopen after close works`() { - val a = note("a", 1) - store.insert(a) - store.close() + fun `reopen after close works`() = + runBlocking { + val a = note("a", 1) + store.insert(a) + store.close() - val reopened = FsEventStore(root) - try { - assertEquals(listOf(a.id), reopened.query(Filter(ids = listOf(a.id))).map { it.id }) - } finally { - reopened.close() + val reopened = FsEventStore(root) + try { + assertEquals(listOf(a.id), reopened.query(Filter(ids = listOf(a.id))).map { it.id }) + } finally { + reopened.close() + } } - } // ------------------------------------------------------------------ // Concurrency — two writer threads serialise cleanly // ------------------------------------------------------------------ @Test - fun `concurrent inserts on two threads are both persisted`() { - val events = (1..20).map { note("n$it", it.toLong()) } - val half = events.size / 2 + fun `concurrent inserts on two threads are both persisted`() = + runBlocking { + val events = (1..20).map { note("n$it", it.toLong()) } + val half = events.size / 2 - val t1 = - Thread { - events.take(half).forEach { store.insert(it) } + // Two real threads via Dispatchers.IO so the in-process lock has to + // arbitrate. join via coroutineScope. + coroutineScope { + launch(Dispatchers.IO) { + events.take(half).forEach { store.insert(it) } + } + launch(Dispatchers.IO) { + events.drop(half).forEach { store.insert(it) } + } } - val t2 = - Thread { - events.drop(half).forEach { store.insert(it) } - } - t1.start() - t2.start() - t1.join() - t2.join() - val got = store.query(Filter(authors = listOf(signer.pubKey))).map { it.id }.toSet() - assertEquals(events.map { it.id }.toSet(), got) - } + val got = store.query(Filter(authors = listOf(signer.pubKey))).map { it.id }.toSet() + assertEquals(events.map { it.id }.toSet(), got) + } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsParityTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsParityTest.kt index 7b18122f2..286c89433 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsParityTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsParityTest.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -81,7 +82,7 @@ class FsParityTest { } /** Insert into both stores. Swallow SQLite rejections (we only care about the resulting state). */ - private fun insertBoth(event: Event) { + private suspend fun insertBoth(event: Event) { try { sqlite.insert(event) } catch (_: Throwable) { @@ -93,7 +94,7 @@ class FsParityTest { } /** Assert both stores return the same ids (as a set) for the given filter. */ - private fun assertParity( + private suspend fun assertParity( filter: Filter, message: String = "", ) { @@ -103,7 +104,7 @@ class FsParityTest { } /** Same, but expect a stable DESC-by-createdAt ordering. */ - private fun assertParityOrdered( + private suspend fun assertParityOrdered( filter: Filter, message: String = "", ) { @@ -135,350 +136,368 @@ class FsParityTest { // ------------------------------------------------------------------ @Test - fun `id lookup matches`() { - val n = note("hello", 10) - insertBoth(n) - assertParity(Filter(ids = listOf(n.id))) - } + fun `id lookup matches`() = + runBlocking { + val n = note("hello", 10) + insertBoth(n) + assertParity(Filter(ids = listOf(n.id))) + } @Test - fun `kind + author query matches`() { - val a = note("a", 1) - val b = note("b", 2) - val c = note("c", 3, s = otherSigner) - listOf(a, b, c).forEach(::insertBoth) + fun `kind + author query matches`() = + runBlocking { + val a = note("a", 1) + val b = note("b", 2) + val c = note("c", 3, s = otherSigner) + listOf(a, b, c).forEach { insertBoth(it) } - assertParityOrdered(Filter(kinds = listOf(1), authors = listOf(signer.pubKey))) - assertParityOrdered(Filter(authors = listOf(signer.pubKey, otherSigner.pubKey))) - } + assertParityOrdered(Filter(kinds = listOf(1), authors = listOf(signer.pubKey))) + assertParityOrdered(Filter(authors = listOf(signer.pubKey, otherSigner.pubKey))) + } @Test - fun `since until limit match`() { - repeat(10) { i -> insertBoth(note("n$i", i.toLong() + 1)) } - assertParityOrdered(Filter(authors = listOf(signer.pubKey), since = 4, until = 8)) - assertParityOrdered(Filter(authors = listOf(signer.pubKey), limit = 3)) - } + fun `since until limit match`() = + runBlocking { + repeat(10) { i -> insertBoth(note("n$i", i.toLong() + 1)) } + assertParityOrdered(Filter(authors = listOf(signer.pubKey), since = 4, until = 8)) + assertParityOrdered(Filter(authors = listOf(signer.pubKey), limit = 3)) + } // ------------------------------------------------------------------ // Tag indexing // ------------------------------------------------------------------ @Test - fun `single-letter tag queries match`() { - val tagged = - signer.sign( - createdAt = 5, - kind = 1, - tags = arrayOf(arrayOf("t", "nostr"), arrayOf("t", "bitcoin")), - content = "x", - ) - val plain = note("plain", 6) - insertBoth(tagged) - insertBoth(plain) + fun `single-letter tag queries match`() = + runBlocking { + val tagged = + signer.sign( + createdAt = 5, + kind = 1, + tags = arrayOf(arrayOf("t", "nostr"), arrayOf("t", "bitcoin")), + content = "x", + ) + val plain = note("plain", 6) + insertBoth(tagged) + insertBoth(plain) - assertParity(Filter(tags = mapOf("t" to listOf("nostr")))) - assertParity(Filter(tags = mapOf("t" to listOf("nostr", "bitcoin")))) - } + assertParity(Filter(tags = mapOf("t" to listOf("nostr")))) + assertParity(Filter(tags = mapOf("t" to listOf("nostr", "bitcoin")))) + } // ------------------------------------------------------------------ // Replaceable / Addressable // ------------------------------------------------------------------ @Test - fun `replaceable newer wins parity`() { - val v1 = - signer.sign( - createdAt = 100, - kind = 0, - tags = emptyArray(), - content = "{\"name\":\"v1\"}", - ) - val v2 = - signer.sign( - createdAt = 200, - kind = 0, - tags = emptyArray(), - content = "{\"name\":\"v2\"}", - ) - insertBoth(v1) - insertBoth(v2) + fun `replaceable newer wins parity`() = + runBlocking { + val v1 = + signer.sign( + createdAt = 100, + kind = 0, + tags = emptyArray(), + content = "{\"name\":\"v1\"}", + ) + val v2 = + signer.sign( + createdAt = 200, + kind = 0, + tags = emptyArray(), + content = "{\"name\":\"v2\"}", + ) + insertBoth(v1) + insertBoth(v2) - assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(0))) - assertParity(Filter(ids = listOf(v1.id))) - } + assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(0))) + assertParity(Filter(ids = listOf(v1.id))) + } @Test - fun `replaceable older rejected parity`() { - val newer = - signer.sign(createdAt = 200, kind = 0, tags = emptyArray(), content = "{\"name\":\"new\"}") - val older = - signer.sign(createdAt = 100, kind = 0, tags = emptyArray(), content = "{\"name\":\"old\"}") - insertBoth(newer) - insertBoth(older) + fun `replaceable older rejected parity`() = + runBlocking { + val newer = + signer.sign(createdAt = 200, kind = 0, tags = emptyArray(), content = "{\"name\":\"new\"}") + val older = + signer.sign(createdAt = 100, kind = 0, tags = emptyArray(), content = "{\"name\":\"old\"}") + insertBoth(newer) + insertBoth(older) - assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(0))) - } + assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(0))) + } @Test - fun `addressable d-tag dedup parity`() { - val v1 = article("intro", "v1", 10) - val v2 = article("intro", "v2", 20) - val v3 = article("about", "bio", 15) - insertBoth(v1) - insertBoth(v2) - insertBoth(v3) + fun `addressable d-tag dedup parity`() = + runBlocking { + val v1 = article("intro", "v1", 10) + val v2 = article("intro", "v2", 20) + val v3 = article("about", "bio", 15) + insertBoth(v1) + insertBoth(v2) + insertBoth(v3) - assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) - } + assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) + } @Test - fun `replaceable same-createdAt lexical id tiebreaker parity`() { - // Two kind-0 events with identical createdAt produce different ids - // because their content differs. NIP-01 says the lexically smaller - // id wins on a tie. Both stores must agree, regardless of insertion - // order. - val a = - signer.sign( - createdAt = 100, - kind = 0, - tags = emptyArray(), - content = "{\"name\":\"a\"}", + fun `replaceable same-createdAt lexical id tiebreaker parity`() = + runBlocking { + // Two kind-0 events with identical createdAt produce different ids + // because their content differs. NIP-01 says the lexically smaller + // id wins on a tie. Both stores must agree, regardless of insertion + // order. + val a = + signer.sign( + createdAt = 100, + kind = 0, + tags = emptyArray(), + content = "{\"name\":\"a\"}", + ) + val b = + signer.sign( + createdAt = 100, + kind = 0, + tags = emptyArray(), + content = "{\"name\":\"b\"}", + ) + + insertBoth(a) + insertBoth(b) + + assertParity( + Filter(authors = listOf(signer.pubKey), kinds = listOf(0)), + "loser-then-winner: lexically smaller id should win", ) - val b = - signer.sign( - createdAt = 100, - kind = 0, - tags = emptyArray(), - content = "{\"name\":\"b\"}", + assertParity( + Filter(ids = listOf(a.id, b.id)), + "the loser must not survive in the by-id query", ) - - insertBoth(a) - insertBoth(b) - - assertParity( - Filter(authors = listOf(signer.pubKey), kinds = listOf(0)), - "loser-then-winner: lexically smaller id should win", - ) - assertParity( - Filter(ids = listOf(a.id, b.id)), - "the loser must not survive in the by-id query", - ) - } + } @Test - fun `addressable same-createdAt lexical id tiebreaker parity`() { - val a = article("tie", "version a", 100) - val b = article("tie", "version b", 100) + fun `addressable same-createdAt lexical id tiebreaker parity`() = + runBlocking { + val a = article("tie", "version a", 100) + val b = article("tie", "version b", 100) - insertBoth(a) - insertBoth(b) + insertBoth(a) + insertBoth(b) - assertParity( - Filter( - authors = listOf(signer.pubKey), - kinds = listOf(LongTextNoteEvent.KIND), - tags = mapOf("d" to listOf("tie")), - ), - ) - assertParity(Filter(ids = listOf(a.id, b.id))) - } + assertParity( + Filter( + authors = listOf(signer.pubKey), + kinds = listOf(LongTextNoteEvent.KIND), + tags = mapOf("d" to listOf("tie")), + ), + ) + assertParity(Filter(ids = listOf(a.id, b.id))) + } // ------------------------------------------------------------------ // Deletion (NIP-09) // ------------------------------------------------------------------ @Test - fun `deletion by id parity`() { - val a = note("a", 10) - val b = note("b", 20) - insertBoth(a) - insertBoth(b) + fun `deletion by id parity`() = + runBlocking { + val a = note("a", 10) + val b = note("b", 20) + insertBoth(a) + insertBoth(b) - val del = signer.sign(DeletionEvent.build(listOf(a), createdAt = 30)) - insertBoth(del) + val del = signer.sign(DeletionEvent.build(listOf(a), createdAt = 30)) + insertBoth(del) - assertParity(Filter(ids = listOf(a.id))) - assertParity(Filter(ids = listOf(b.id))) - assertParity(Filter(kinds = listOf(DeletionEvent.KIND))) + assertParity(Filter(ids = listOf(a.id))) + assertParity(Filter(ids = listOf(b.id))) + assertParity(Filter(kinds = listOf(DeletionEvent.KIND))) - // Re-insert blocked. - insertBoth(a) - assertParity(Filter(ids = listOf(a.id))) - } + // Re-insert blocked. + insertBoth(a) + assertParity(Filter(ids = listOf(a.id))) + } @Test - fun `deletion by address parity`() { - val v = article("intro", "v1", 10) - insertBoth(v) + fun `deletion by address parity`() = + runBlocking { + val v = article("intro", "v1", 10) + insertBoth(v) - val del = signer.sign(DeletionEvent.build(listOf(v), createdAt = 20)) - insertBoth(del) + val del = signer.sign(DeletionEvent.build(listOf(v), createdAt = 20)) + insertBoth(del) - assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) + assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) - // Older event at this address must be blocked, newer must pass. - insertBoth(article("intro", "older", 5)) - insertBoth(article("intro", "newer", 100)) + // Older event at this address must be blocked, newer must pass. + insertBoth(article("intro", "older", 5)) + insertBoth(article("intro", "newer", 100)) - assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) - } + assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) + } // ------------------------------------------------------------------ // Expiration (NIP-40) // ------------------------------------------------------------------ @Test - fun `expiration sweep parity`() { - // Build events with future-then-past expirations relative to now. - val now = - com.vitorpamplona.quartz.utils.TimeUtils - .now() - val expired = - signer.sign( - createdAt = now - 100, - kind = 1, - tags = arrayOf(arrayOf("expiration", (now - 50).toString())), - content = "old", - ) - val alive = - signer.sign( - createdAt = now - 100, - kind = 1, - tags = arrayOf(arrayOf("expiration", (now + 1_000_000).toString())), - content = "still here", - ) - insertBoth(expired) // both stores reject (already expired) - insertBoth(alive) + fun `expiration sweep parity`() = + runBlocking { + // Build events with future-then-past expirations relative to now. + val now = + com.vitorpamplona.quartz.utils.TimeUtils + .now() + val expired = + signer.sign( + createdAt = now - 100, + kind = 1, + tags = arrayOf(arrayOf("expiration", (now - 50).toString())), + content = "old", + ) + val alive = + signer.sign( + createdAt = now - 100, + kind = 1, + tags = arrayOf(arrayOf("expiration", (now + 1_000_000).toString())), + content = "still here", + ) + insertBoth(expired) // both stores reject (already expired) + insertBoth(alive) - assertParity(Filter(ids = listOf(expired.id))) - assertParity(Filter(ids = listOf(alive.id))) + assertParity(Filter(ids = listOf(expired.id))) + assertParity(Filter(ids = listOf(alive.id))) - // Sweep both; alive survives. - sqlite.deleteExpiredEvents() - fs.deleteExpiredEvents() - assertParity(Filter(ids = listOf(alive.id))) - } + // Sweep both; alive survives. + sqlite.deleteExpiredEvents() + fs.deleteExpiredEvents() + assertParity(Filter(ids = listOf(alive.id))) + } // ------------------------------------------------------------------ // Search (NIP-50) // ------------------------------------------------------------------ @Test - fun `search parity`() { - val a = note("hello bitcoin", 1) - val b = note("nostr only", 2) - val c = note("bitcoin and nostr", 3) - insertBoth(a) - insertBoth(b) - insertBoth(c) + fun `search parity`() = + runBlocking { + val a = note("hello bitcoin", 1) + val b = note("nostr only", 2) + val c = note("bitcoin and nostr", 3) + insertBoth(a) + insertBoth(b) + insertBoth(c) - // Tokenizers differ slightly between SQLite FTS5 unicode61 and - // our Kotlin port, so we stick to plain ASCII single-token queries - // where both should agree. - assertParity(Filter(search = "bitcoin")) - assertParity(Filter(search = "nostr")) - } + // Tokenizers differ slightly between SQLite FTS5 unicode61 and + // our Kotlin port, so we stick to plain ASCII single-token queries + // where both should agree. + assertParity(Filter(search = "bitcoin")) + assertParity(Filter(search = "nostr")) + } // ------------------------------------------------------------------ // Count // ------------------------------------------------------------------ @Test - fun `count parity across mixed stream`() { - listOf( - note("a", 1), - note("b", 2), - note("c", 3), - note("from-other", 4, s = otherSigner), - ).forEach(::insertBoth) + fun `count parity across mixed stream`() = + runBlocking { + listOf( + note("a", 1), + note("b", 2), + note("c", 3), + note("from-other", 4, s = otherSigner), + ).forEach { insertBoth(it) } - val filter = Filter(authors = listOf(signer.pubKey)) - assertEquals(sqlite.count(filter), fs.count(filter)) - } + val filter = Filter(authors = listOf(signer.pubKey)) + assertEquals(sqlite.count(filter), fs.count(filter)) + } // ------------------------------------------------------------------ // Mixed kitchen-sink scenario // ------------------------------------------------------------------ @Test - fun `kitchen sink scenario`() { - // Notes - val n1 = note("first", 1) - val n2 = note("second", 2) - // Replaceable - val meta1 = - signer.sign(createdAt = 10, kind = 0, tags = emptyArray(), content = "{\"name\":\"v1\"}") - val meta2 = - signer.sign(createdAt = 20, kind = 0, tags = emptyArray(), content = "{\"name\":\"v2\"}") - // Addressable - val artA = article("a", "A v1", 30) - val artB = article("b", "B v1", 30) - val artBv2 = article("b", "B v2", 50) - // Deletion of n1 - val del = signer.sign(DeletionEvent.build(listOf(n1), createdAt = 40)) + fun `kitchen sink scenario`() = + runBlocking { + // Notes + val n1 = note("first", 1) + val n2 = note("second", 2) + // Replaceable + val meta1 = + signer.sign(createdAt = 10, kind = 0, tags = emptyArray(), content = "{\"name\":\"v1\"}") + val meta2 = + signer.sign(createdAt = 20, kind = 0, tags = emptyArray(), content = "{\"name\":\"v2\"}") + // Addressable + val artA = article("a", "A v1", 30) + val artB = article("b", "B v1", 30) + val artBv2 = article("b", "B v2", 50) + // Deletion of n1 + val del = signer.sign(DeletionEvent.build(listOf(n1), createdAt = 40)) - listOf(n1, n2, meta1, meta2, artA, artB, artBv2, del).forEach(::insertBoth) + listOf(n1, n2, meta1, meta2, artA, artB, artBv2, del).forEach { insertBoth(it) } - // Snapshots that should match. - assertParity(Filter(ids = listOf(n1.id)), "n1 deleted") - assertParity(Filter(ids = listOf(n2.id)), "n2 alive") - assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(0)), "metadata winner") - assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND)), "articles set") - assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(DeletionEvent.KIND)), "deletion present") - } + // Snapshots that should match. + assertParity(Filter(ids = listOf(n1.id)), "n1 deleted") + assertParity(Filter(ids = listOf(n2.id)), "n2 alive") + assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(0)), "metadata winner") + assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND)), "articles set") + assertParity(Filter(authors = listOf(signer.pubKey), kinds = listOf(DeletionEvent.KIND)), "deletion present") + } // ------------------------------------------------------------------ // Multi-filter union // ------------------------------------------------------------------ @Test - fun `multi-filter union parity`() { - val a = note("a", 1) - val b = note("b", 2, s = otherSigner) - insertBoth(a) - insertBoth(b) + fun `multi-filter union parity`() = + runBlocking { + val a = note("a", 1) + val b = note("b", 2, s = otherSigner) + insertBoth(a) + insertBoth(b) - val filters = - listOf( - Filter(authors = listOf(signer.pubKey)), - Filter(authors = listOf(otherSigner.pubKey)), + val filters = + listOf( + Filter(authors = listOf(signer.pubKey)), + Filter(authors = listOf(otherSigner.pubKey)), + ) + + assertEquals( + sqlite.query(filters).map { it.id }.toSet(), + fs.query(filters).map { it.id }.toSet(), ) - - assertEquals( - sqlite.query(filters).map { it.id }.toSet(), - fs.query(filters).map { it.id }.toSet(), - ) - } + } // ------------------------------------------------------------------ // Direct delete by filter // ------------------------------------------------------------------ @Test - fun `delete by filter parity`() { - val toKill = note("dead", 5) - val survivor = note("alive", 6) - insertBoth(toKill) - insertBoth(survivor) + fun `delete by filter parity`() = + runBlocking { + val toKill = note("dead", 5) + val survivor = note("alive", 6) + insertBoth(toKill) + insertBoth(survivor) - sqlite.delete(Filter(ids = listOf(toKill.id))) - fs.delete(Filter(ids = listOf(toKill.id))) + sqlite.delete(Filter(ids = listOf(toKill.id))) + fs.delete(Filter(ids = listOf(toKill.id))) - assertParity(Filter(authors = listOf(signer.pubKey))) - } + assertParity(Filter(authors = listOf(signer.pubKey))) + } // ------------------------------------------------------------------ // Helper: ensure SQLite store really does what we think // ------------------------------------------------------------------ @Test - fun `helper sanity - empty stores agree`() { - assertParity(Filter(authors = listOf(signer.pubKey))) - assertParity(Filter(kinds = listOf(1))) - } + fun `helper sanity - empty stores agree`() = + runBlocking { + assertParity(Filter(authors = listOf(signer.pubKey))) + assertParity(Filter(kinds = listOf(1))) + } @Suppress("unused") - private fun debugDump(label: String): String { + private suspend fun debugDump(label: String): String { val sqIds = sqlite .query(Filter(authors = listOf(signer.pubKey, otherSigner.pubKey))) diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryTest.kt index 42989502a..3d051fd63 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryTest.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -71,351 +72,372 @@ class FsQueryTest { // ------------------------------------------------------------------ @Test - fun `results ordered by created_at DESC`() { - val a = signA("a", 1) - val b = signA("b", 3) - val c = signA("c", 2) - listOf(a, b, c).forEach(store::insert) + fun `results ordered by created_at DESC`() = + runBlocking { + val a = signA("a", 1) + val b = signA("b", 3) + val c = signA("c", 2) + listOf(a, b, c).forEach { store.insert(it) } - val got = store.query(Filter(authors = listOf(signerA.pubKey))) - assertEquals(listOf(b.id, c.id, a.id), got.map { it.id }) - } + val got = store.query(Filter(authors = listOf(signerA.pubKey))) + assertEquals(listOf(b.id, c.id, a.id), got.map { it.id }) + } @Test - fun `limit caps the result count`() { - repeat(5) { i -> store.insert(signA("n$i", i.toLong() + 1)) } - val got = store.query(Filter(authors = listOf(signerA.pubKey), limit = 2)) - assertEquals(2, got.size) - // Highest timestamps come first. - assertEquals("n4", got[0].content) - assertEquals("n3", got[1].content) - } + fun `limit caps the result count`() = + runBlocking { + repeat(5) { i -> store.insert(signA("n$i", i.toLong() + 1)) } + val got = store.query(Filter(authors = listOf(signerA.pubKey), limit = 2)) + assertEquals(2, got.size) + // Highest timestamps come first. + assertEquals("n4", got[0].content) + assertEquals("n3", got[1].content) + } @Test - fun `limit of zero returns empty`() { - store.insert(signA("x", 1)) - assertEquals(emptyList(), store.query(Filter(authors = listOf(signerA.pubKey), limit = 0))) - } + fun `limit of zero returns empty`() = + runBlocking { + store.insert(signA("x", 1)) + assertEquals(emptyList(), store.query(Filter(authors = listOf(signerA.pubKey), limit = 0))) + } // ------------------------------------------------------------------ // Author / kind drivers // ------------------------------------------------------------------ @Test - fun `author filter isolates one user`() { - val a = signA("from-a", 1) - val b = signB("from-b", 2) - store.insert(a) - store.insert(b) + fun `author filter isolates one user`() = + runBlocking { + val a = signA("from-a", 1) + val b = signB("from-b", 2) + store.insert(a) + store.insert(b) - val got = store.query(Filter(authors = listOf(signerA.pubKey))) - assertEquals(listOf(a.id), got.map { it.id }) - } + val got = store.query(Filter(authors = listOf(signerA.pubKey))) + assertEquals(listOf(a.id), got.map { it.id }) + } @Test - fun `author filter with multiple authors unions them`() { - val a = signA("a", 1) - val b = signB("b", 2) - store.insert(a) - store.insert(b) + fun `author filter with multiple authors unions them`() = + runBlocking { + val a = signA("a", 1) + val b = signB("b", 2) + store.insert(a) + store.insert(b) - val got = store.query(Filter(authors = listOf(signerA.pubKey, signerB.pubKey))) - assertEquals(setOf(a.id, b.id), got.map { it.id }.toSet()) - } + val got = store.query(Filter(authors = listOf(signerA.pubKey, signerB.pubKey))) + assertEquals(setOf(a.id, b.id), got.map { it.id }.toSet()) + } @Test - fun `kind filter returns only the requested kinds`() { - // Build two events of different kinds. - val note = signA("note", 1) - val ephemeralKinds = signerA.sign(createdAt = 2, kind = 30023, tags = arrayOf(arrayOf("d", "slug")), content = "article") - store.insert(note) - store.insert(ephemeralKinds) + fun `kind filter returns only the requested kinds`() = + runBlocking { + // Build two events of different kinds. + val note = signA("note", 1) + val ephemeralKinds = signerA.sign(createdAt = 2, kind = 30023, tags = arrayOf(arrayOf("d", "slug")), content = "article") + store.insert(note) + store.insert(ephemeralKinds) - val onlyNotes = store.query(Filter(kinds = listOf(1))) - assertEquals(listOf(note.id), onlyNotes.map { it.id }) + val onlyNotes = store.query(Filter(kinds = listOf(1))) + assertEquals(listOf(note.id), onlyNotes.map { it.id }) - val onlyArticles = store.query(Filter(kinds = listOf(30023))) - assertEquals(listOf(ephemeralKinds.id), onlyArticles.map { it.id }) - } + val onlyArticles = store.query(Filter(kinds = listOf(30023))) + assertEquals(listOf(ephemeralKinds.id), onlyArticles.map { it.id }) + } @Test - fun `kind + author intersect via post-filter`() { - val a = signA("a", 1) - val b = signB("b", 2) - store.insert(a) - store.insert(b) + fun `kind + author intersect via post-filter`() = + runBlocking { + val a = signA("a", 1) + val b = signB("b", 2) + store.insert(a) + store.insert(b) - val got = store.query(Filter(kinds = listOf(1), authors = listOf(signerA.pubKey))) - assertEquals(listOf(a.id), got.map { it.id }) - } + val got = store.query(Filter(kinds = listOf(1), authors = listOf(signerA.pubKey))) + assertEquals(listOf(a.id), got.map { it.id }) + } // ------------------------------------------------------------------ // Tag driver // ------------------------------------------------------------------ @Test - fun `tag filter matches single-letter tags`() { - val tagged = - signerA.sign( - createdAt = 10, - kind = 1, - tags = arrayOf(arrayOf("t", "nostr"), arrayOf("t", "bitcoin")), - content = "tagged", - ) - val untagged = signA("plain", 5) - store.insert(tagged) - store.insert(untagged) + fun `tag filter matches single-letter tags`() = + runBlocking { + val tagged = + signerA.sign( + createdAt = 10, + kind = 1, + tags = arrayOf(arrayOf("t", "nostr"), arrayOf("t", "bitcoin")), + content = "tagged", + ) + val untagged = signA("plain", 5) + store.insert(tagged) + store.insert(untagged) - val got = store.query(Filter(tags = mapOf("t" to listOf("nostr")))) - assertEquals(listOf(tagged.id), got.map { it.id }) - } + val got = store.query(Filter(tags = mapOf("t" to listOf("nostr")))) + assertEquals(listOf(tagged.id), got.map { it.id }) + } @Test - fun `tag OR within key returns union`() { - val t1 = signerA.sign(createdAt = 1, kind = 1, tags = arrayOf(arrayOf("t", "nostr")), content = "n") - val t2 = signerA.sign(createdAt = 2, kind = 1, tags = arrayOf(arrayOf("t", "bitcoin")), content = "b") - val t3 = signerA.sign(createdAt = 3, kind = 1, tags = arrayOf(arrayOf("t", "other")), content = "o") - listOf(t1, t2, t3).forEach(store::insert) + fun `tag OR within key returns union`() = + runBlocking { + val t1 = signerA.sign(createdAt = 1, kind = 1, tags = arrayOf(arrayOf("t", "nostr")), content = "n") + val t2 = signerA.sign(createdAt = 2, kind = 1, tags = arrayOf(arrayOf("t", "bitcoin")), content = "b") + val t3 = signerA.sign(createdAt = 3, kind = 1, tags = arrayOf(arrayOf("t", "other")), content = "o") + listOf(t1, t2, t3).forEach { store.insert(it) } - val got = store.query(Filter(tags = mapOf("t" to listOf("nostr", "bitcoin")))) - assertEquals(setOf(t1.id, t2.id), got.map { it.id }.toSet()) - } + val got = store.query(Filter(tags = mapOf("t" to listOf("nostr", "bitcoin")))) + assertEquals(setOf(t1.id, t2.id), got.map { it.id }.toSet()) + } @Test - fun `tagsAll across keys requires all matches`() { - val both = - signerA.sign( - createdAt = 1, - kind = 1, - tags = arrayOf(arrayOf("t", "nostr"), arrayOf("e", "a".repeat(64))), - content = "both", - ) - val onlyT = signerA.sign(createdAt = 2, kind = 1, tags = arrayOf(arrayOf("t", "nostr")), content = "t-only") - val onlyE = signerA.sign(createdAt = 3, kind = 1, tags = arrayOf(arrayOf("e", "a".repeat(64))), content = "e-only") - listOf(both, onlyT, onlyE).forEach(store::insert) + fun `tagsAll across keys requires all matches`() = + runBlocking { + val both = + signerA.sign( + createdAt = 1, + kind = 1, + tags = arrayOf(arrayOf("t", "nostr"), arrayOf("e", "a".repeat(64))), + content = "both", + ) + val onlyT = signerA.sign(createdAt = 2, kind = 1, tags = arrayOf(arrayOf("t", "nostr")), content = "t-only") + val onlyE = signerA.sign(createdAt = 3, kind = 1, tags = arrayOf(arrayOf("e", "a".repeat(64))), content = "e-only") + listOf(both, onlyT, onlyE).forEach { store.insert(it) } - val got = - store.query( - Filter( - tagsAll = mapOf("t" to listOf("nostr"), "e" to listOf("a".repeat(64))), - ), - ) - assertEquals(listOf(both.id), got.map { it.id }) - } + val got = + store.query( + Filter( + tagsAll = mapOf("t" to listOf("nostr"), "e" to listOf("a".repeat(64))), + ), + ) + assertEquals(listOf(both.id), got.map { it.id }) + } // ------------------------------------------------------------------ // Tag-value directory naming: raw when fs-safe, _h_ otherwise. // ------------------------------------------------------------------ @Test - fun `safe ASCII tag values get raw directory names`() { - val e = - signerA.sign( - createdAt = 10, - kind = 1, - tags = arrayOf(arrayOf("t", "nostr")), - content = "x", - ) - store.insert(e) - // The raw value is the directory name — directly inspectable. - val rawDir = root.resolve("idx/tag/t/nostr") - assertTrue(rawDir.exists(), "ASCII-safe tag should land in idx/tag/t/nostr/") - assertEquals(1, rawDir.listDirectoryEntries().size) - } + fun `safe ASCII tag values get raw directory names`() = + runBlocking { + val e = + signerA.sign( + createdAt = 10, + kind = 1, + tags = arrayOf(arrayOf("t", "nostr")), + content = "x", + ) + store.insert(e) + // The raw value is the directory name — directly inspectable. + val rawDir = root.resolve("idx/tag/t/nostr") + assertTrue(rawDir.exists(), "ASCII-safe tag should land in idx/tag/t/nostr/") + assertEquals(1, rawDir.listDirectoryEntries().size) + } @Test - fun `pubkey p-tag uses raw 64-hex directory name`() { - // The motivating case: notifications. p-tags pointing at a - // pubkey land under idx/tag/p// — no hash, directly - // ls-able. - val target = signerB.pubKey - val e = - signerA.sign( - createdAt = 5, - kind = 1, - tags = arrayOf(arrayOf("p", target)), - content = "@you", - ) - store.insert(e) - val pDir = root.resolve("idx/tag/p/$target") - assertTrue(pDir.exists(), "p-tag pubkey should be ls-able directly: idx/tag/p/$target/") - assertEquals(1, pDir.listDirectoryEntries().size) - } + fun `pubkey p-tag uses raw 64-hex directory name`() = + runBlocking { + // The motivating case: notifications. p-tags pointing at a + // pubkey land under idx/tag/p// — no hash, directly + // ls-able. + val target = signerB.pubKey + val e = + signerA.sign( + createdAt = 5, + kind = 1, + tags = arrayOf(arrayOf("p", target)), + content = "@you", + ) + store.insert(e) + val pDir = root.resolve("idx/tag/p/$target") + assertTrue(pDir.exists(), "p-tag pubkey should be ls-able directly: idx/tag/p/$target/") + assertEquals(1, pDir.listDirectoryEntries().size) + } @Test - fun `tag value with emoji falls back to hashed directory name`() { - val e = - signerA.sign( - createdAt = 10, - kind = 1, - tags = arrayOf(arrayOf("t", "🔥")), - content = "x", - ) - store.insert(e) - // Exactly one entry under t/ and it must be in the _h_ hash - // bucket — emoji is not fs-safe. - val tDir = root.resolve("idx/tag/t") - val entries = tDir.listDirectoryEntries().map { it.fileName.toString() } - assertEquals(1, entries.size, "expected one bucket dir, got: $entries") - assertTrue(entries.single().startsWith("_h_"), "emoji tag must hash; got '${entries.single()}'") - } + fun `tag value with emoji falls back to hashed directory name`() = + runBlocking { + val e = + signerA.sign( + createdAt = 10, + kind = 1, + tags = arrayOf(arrayOf("t", "🔥")), + content = "x", + ) + store.insert(e) + // Exactly one entry under t/ and it must be in the _h_ hash + // bucket — emoji is not fs-safe. + val tDir = root.resolve("idx/tag/t") + val entries = tDir.listDirectoryEntries().map { it.fileName.toString() } + assertEquals(1, entries.size, "expected one bucket dir, got: $entries") + assertTrue(entries.single().startsWith("_h_"), "emoji tag must hash; got '${entries.single()}'") + } @Test - fun `tag value containing a slash falls back to hashed directory name`() { - val e = - signerA.sign( - createdAt = 10, - kind = 1, - tags = arrayOf(arrayOf("r", "https://example.com/page")), - content = "x", - ) - store.insert(e) - val rDir = root.resolve("idx/tag/r") - val entries = rDir.listDirectoryEntries().map { it.fileName.toString() } - assertEquals(1, entries.size, "expected one bucket dir, got: $entries") - assertTrue(entries.single().startsWith("_h_"), "URL tag must hash; got '${entries.single()}'") - } + fun `tag value containing a slash falls back to hashed directory name`() = + runBlocking { + val e = + signerA.sign( + createdAt = 10, + kind = 1, + tags = arrayOf(arrayOf("r", "https://example.com/page")), + content = "x", + ) + store.insert(e) + val rDir = root.resolve("idx/tag/r") + val entries = rDir.listDirectoryEntries().map { it.fileName.toString() } + assertEquals(1, entries.size, "expected one bucket dir, got: $entries") + assertTrue(entries.single().startsWith("_h_"), "URL tag must hash; got '${entries.single()}'") + } @Test - fun `query round-trips for both raw and hashed values`() { - // Each query must use the same naming rule as the writer or it - // walks a directory that doesn't exist. Insert both a raw-safe - // and a hash-required tag and verify they're both findable. - val safe = - signerA.sign( - createdAt = 1, - kind = 1, - tags = arrayOf(arrayOf("t", "nostr")), - content = "safe", + fun `query round-trips for both raw and hashed values`() = + runBlocking { + // Each query must use the same naming rule as the writer or it + // walks a directory that doesn't exist. Insert both a raw-safe + // and a hash-required tag and verify they're both findable. + val safe = + signerA.sign( + createdAt = 1, + kind = 1, + tags = arrayOf(arrayOf("t", "nostr")), + content = "safe", + ) + val unsafe = + signerA.sign( + createdAt = 2, + kind = 1, + tags = arrayOf(arrayOf("t", "🔥")), + content = "unsafe", + ) + store.insert(safe) + store.insert(unsafe) + assertEquals( + listOf(safe.id), + store.query(Filter(tags = mapOf("t" to listOf("nostr")))).map { it.id }, ) - val unsafe = - signerA.sign( - createdAt = 2, - kind = 1, - tags = arrayOf(arrayOf("t", "🔥")), - content = "unsafe", + assertEquals( + listOf(unsafe.id), + store.query(Filter(tags = mapOf("t" to listOf("🔥")))).map { it.id }, ) - store.insert(safe) - store.insert(unsafe) - assertEquals( - listOf(safe.id), - store.query(Filter(tags = mapOf("t" to listOf("nostr")))).map { it.id }, - ) - assertEquals( - listOf(unsafe.id), - store.query(Filter(tags = mapOf("t" to listOf("🔥")))).map { it.id }, - ) - } + } @Test - fun `non-single-letter tags are not reverse-indexed`() { - // SQLite parity: DefaultIndexingStrategy only indexes single-letter - // tag names, so a tag-driven query for `mytag = foo` finds no - // candidates. The event is still persisted and can be fetched via - // id / author / kind — just not via a reverse tag lookup. - val e = - signerA.sign( - createdAt = 1, - kind = 1, - tags = arrayOf(arrayOf("mytag", "foo")), - content = "x", - ) - store.insert(e) + fun `non-single-letter tags are not reverse-indexed`() = + runBlocking { + // SQLite parity: DefaultIndexingStrategy only indexes single-letter + // tag names, so a tag-driven query for `mytag = foo` finds no + // candidates. The event is still persisted and can be fetched via + // id / author / kind — just not via a reverse tag lookup. + val e = + signerA.sign( + createdAt = 1, + kind = 1, + tags = arrayOf(arrayOf("mytag", "foo")), + content = "x", + ) + store.insert(e) - assertEquals(emptyList(), store.query(Filter(tags = mapOf("mytag" to listOf("foo")))).map { it.id }) - assertEquals(listOf(e.id), store.query(Filter(authors = listOf(signerA.pubKey))).map { it.id }) - assertEquals(listOf(e.id), store.query(Filter(ids = listOf(e.id))).map { it.id }) - } + assertEquals(emptyList(), store.query(Filter(tags = mapOf("mytag" to listOf("foo")))).map { it.id }) + assertEquals(listOf(e.id), store.query(Filter(authors = listOf(signerA.pubKey))).map { it.id }) + assertEquals(listOf(e.id), store.query(Filter(ids = listOf(e.id))).map { it.id }) + } // ------------------------------------------------------------------ // since / until // ------------------------------------------------------------------ @Test - fun `since and until window filter`() { - val e1 = signA("t1", 100) - val e2 = signA("t2", 200) - val e3 = signA("t3", 300) - listOf(e1, e2, e3).forEach(store::insert) + fun `since and until window filter`() = + runBlocking { + val e1 = signA("t1", 100) + val e2 = signA("t2", 200) + val e3 = signA("t3", 300) + listOf(e1, e2, e3).forEach { store.insert(it) } - val got = store.query(Filter(since = 150, until = 250)) - assertEquals(listOf(e2.id), got.map { it.id }) - } + val got = store.query(Filter(since = 150, until = 250)) + assertEquals(listOf(e2.id), got.map { it.id }) + } // ------------------------------------------------------------------ // count // ------------------------------------------------------------------ @Test - fun `count matches query size`() { - repeat(4) { i -> store.insert(signA("n$i", i.toLong() + 1)) } - val filter = Filter(authors = listOf(signerA.pubKey)) - assertEquals(store.query(filter).size, store.count(filter)) - } + fun `count matches query size`() = + runBlocking { + repeat(4) { i -> store.insert(signA("n$i", i.toLong() + 1)) } + val filter = Filter(authors = listOf(signerA.pubKey)) + assertEquals(store.query(filter).size, store.count(filter)) + } // ------------------------------------------------------------------ // Index hardlink maintenance // ------------------------------------------------------------------ @Test - fun `insert creates hardlinks in every expected index dir`() { - val tagged = - signerA.sign( - createdAt = 42, - kind = 1, - tags = arrayOf(arrayOf("t", "nostr")), - content = "x", - ) - store.insert(tagged) + fun `insert creates hardlinks in every expected index dir`() = + runBlocking { + val tagged = + signerA.sign( + createdAt = 42, + kind = 1, + tags = arrayOf(arrayOf("t", "nostr")), + content = "x", + ) + store.insert(tagged) - val kindDir = root.resolve("idx/kind/1") - val authorDir = root.resolve("idx/author/${signerA.pubKey}") - assertTrue(kindDir.exists() && kindDir.listDirectoryEntries().size == 1, "kind index missing") - assertTrue(authorDir.exists() && authorDir.listDirectoryEntries().size == 1, "author index missing") - val tagNameDir = root.resolve("idx/tag/t") - assertTrue(tagNameDir.exists(), "tag 't' dir missing") - val tagValueDirs = tagNameDir.listDirectoryEntries() - assertEquals(1, tagValueDirs.size, "exactly one tag-value subdir expected") - assertEquals(1, tagValueDirs[0].listDirectoryEntries().size, "tag-value dir should contain one entry") - } + val kindDir = root.resolve("idx/kind/1") + val authorDir = root.resolve("idx/author/${signerA.pubKey}") + assertTrue(kindDir.exists() && kindDir.listDirectoryEntries().size == 1, "kind index missing") + assertTrue(authorDir.exists() && authorDir.listDirectoryEntries().size == 1, "author index missing") + val tagNameDir = root.resolve("idx/tag/t") + assertTrue(tagNameDir.exists(), "tag 't' dir missing") + val tagValueDirs = tagNameDir.listDirectoryEntries() + assertEquals(1, tagValueDirs.size, "exactly one tag-value subdir expected") + assertEquals(1, tagValueDirs[0].listDirectoryEntries().size, "tag-value dir should contain one entry") + } @Test - fun `delete removes hardlinks so directories become empty`() { - val e = signerA.sign(createdAt = 1, kind = 1, tags = arrayOf(arrayOf("t", "nostr")), content = "x") - store.insert(e) - store.delete(e.id) + fun `delete removes hardlinks so directories become empty`() = + runBlocking { + val e = signerA.sign(createdAt = 1, kind = 1, tags = arrayOf(arrayOf("t", "nostr")), content = "x") + store.insert(e) + store.delete(e.id) - val kindDir = root.resolve("idx/kind/1") - val authorDir = root.resolve("idx/author/${signerA.pubKey}") - val tagValueDirs = - root - .resolve("idx/tag/t") - .takeIf { it.exists() } - ?.listDirectoryEntries() - .orEmpty() + val kindDir = root.resolve("idx/kind/1") + val authorDir = root.resolve("idx/author/${signerA.pubKey}") + val tagValueDirs = + root + .resolve("idx/tag/t") + .takeIf { it.exists() } + ?.listDirectoryEntries() + .orEmpty() - // Directories may remain as empty husks — what matters is the entries are gone. - if (kindDir.exists()) assertEquals(0, kindDir.listDirectoryEntries().size, "kind entry leaked") - if (authorDir.exists()) assertEquals(0, authorDir.listDirectoryEntries().size, "author entry leaked") - tagValueDirs.forEach { assertEquals(0, it.listDirectoryEntries().size, "tag entry leaked") } - } + // Directories may remain as empty husks — what matters is the entries are gone. + if (kindDir.exists()) assertEquals(0, kindDir.listDirectoryEntries().size, "kind entry leaked") + if (authorDir.exists()) assertEquals(0, authorDir.listDirectoryEntries().size, "author entry leaked") + tagValueDirs.forEach { assertEquals(0, it.listDirectoryEntries().size, "tag entry leaked") } + } // ------------------------------------------------------------------ // Seed persistence across reopen // ------------------------------------------------------------------ @Test - fun `reopening the store preserves queryability`() { - val tagged = signerA.sign(createdAt = 1, kind = 1, tags = arrayOf(arrayOf("t", "nostr")), content = "x") - store.insert(tagged) - store.close() + fun `reopening the store preserves queryability`() = + runBlocking { + val tagged = signerA.sign(createdAt = 1, kind = 1, tags = arrayOf(arrayOf("t", "nostr")), content = "x") + store.insert(tagged) + store.close() - val reopened = FsEventStore(root) - try { - val got = reopened.query(Filter(tags = mapOf("t" to listOf("nostr")))) - assertEquals(listOf(tagged.id), got.map { it.id }) - } finally { - reopened.close() + val reopened = FsEventStore(root) + try { + val got = reopened.query(Filter(tags = mapOf("t" to listOf("nostr")))) + assertEquals(listOf(tagged.id), got.map { it.id }) + } finally { + reopened.close() + } } - } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt index 201efaad3..3678d795c 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSearchTest.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -66,193 +67,209 @@ class FsSearchTest { // ------------------------------------------------------------------ @Test - fun `tokenizer splits on whitespace and punctuation`() { - assertEquals(setOf("hello", "world"), FsSearchTokenizer.tokenize("hello, world!")) - } + fun `tokenizer splits on whitespace and punctuation`() = + runBlocking { + assertEquals(setOf("hello", "world"), FsSearchTokenizer.tokenize("hello, world!")) + } @Test - fun `tokenizer is case insensitive`() { - assertEquals(FsSearchTokenizer.tokenize("Bitcoin"), FsSearchTokenizer.tokenize("BITCOIN")) - assertEquals(FsSearchTokenizer.tokenize("Bitcoin"), FsSearchTokenizer.tokenize("bitcoin")) - } + fun `tokenizer is case insensitive`() = + runBlocking { + assertEquals(FsSearchTokenizer.tokenize("Bitcoin"), FsSearchTokenizer.tokenize("BITCOIN")) + assertEquals(FsSearchTokenizer.tokenize("Bitcoin"), FsSearchTokenizer.tokenize("bitcoin")) + } @Test - fun `tokenizer handles empty and punctuation-only strings`() { - assertEquals(emptySet(), FsSearchTokenizer.tokenize("")) - assertEquals(emptySet(), FsSearchTokenizer.tokenize("...")) - assertEquals(emptySet(), FsSearchTokenizer.tokenize(" ")) - } + fun `tokenizer handles empty and punctuation-only strings`() = + runBlocking { + assertEquals(emptySet(), FsSearchTokenizer.tokenize("")) + assertEquals(emptySet(), FsSearchTokenizer.tokenize("...")) + assertEquals(emptySet(), FsSearchTokenizer.tokenize(" ")) + } @Test - fun `tokenizer keeps unicode letters`() { - assertEquals(setOf("café", "über"), FsSearchTokenizer.tokenize("Café Über")) - } + fun `tokenizer keeps unicode letters`() = + runBlocking { + assertEquals(setOf("café", "über"), FsSearchTokenizer.tokenize("Café Über")) + } // ------------------------------------------------------------------ // Index maintenance // ------------------------------------------------------------------ @Test - fun `searchable event creates one fts entry per unique token`() { - val n = note("bitcoin nostr bitcoin", ts = 100) - store.insert(n) + fun `searchable event creates one fts entry per unique token`() = + runBlocking { + val n = note("bitcoin nostr bitcoin", ts = 100) + store.insert(n) - val ftsRoot = root.resolve("idx/fts") - val tokenDirs = ftsRoot.listDirectoryEntries().map { it.fileName.toString() }.toSet() - // TextNoteEvent.indexableContent() prepends a "Subject: " prefix so - // we get the content tokens plus the subject ones. What matters is - // that each unique token yields exactly one entry under its dir. - assertTrue("bitcoin" in tokenDirs) - assertTrue("nostr" in tokenDirs) - assertEquals(1, ftsRoot.resolve("bitcoin").listDirectoryEntries().size) - assertEquals(1, ftsRoot.resolve("nostr").listDirectoryEntries().size) - } - - @Test - fun `non-searchable event does not produce fts entries`() { - val meta = - signer.sign( - createdAt = 1, - kind = MetadataEvent.KIND, - tags = emptyArray(), - content = "{\"name\":\"vitor\"}", - ) - store.insert(meta) - - val ftsRoot = root.resolve("idx/fts") - assertEquals(0, ftsRoot.listDirectoryEntries().size, "MetadataEvent is not SearchableEvent") - } - - @Test - fun `delete removes fts entries`() { - val n = note("bitcoin nostr", ts = 100) - store.insert(n) - store.delete(n.id) - - val ftsRoot = root.resolve("idx/fts") - // Token directories may remain as empty husks. - for (tokenDir in ftsRoot.listDirectoryEntries()) { - assertEquals(0, tokenDir.listDirectoryEntries().size, "token entry leaked: $tokenDir") + val ftsRoot = root.resolve("idx/fts") + val tokenDirs = ftsRoot.listDirectoryEntries().map { it.fileName.toString() }.toSet() + // TextNoteEvent.indexableContent() prepends a "Subject: " prefix so + // we get the content tokens plus the subject ones. What matters is + // that each unique token yields exactly one entry under its dir. + assertTrue("bitcoin" in tokenDirs) + assertTrue("nostr" in tokenDirs) + assertEquals(1, ftsRoot.resolve("bitcoin").listDirectoryEntries().size) + assertEquals(1, ftsRoot.resolve("nostr").listDirectoryEntries().size) + } + + @Test + fun `non-searchable event does not produce fts entries`() = + runBlocking { + val meta = + signer.sign( + createdAt = 1, + kind = MetadataEvent.KIND, + tags = emptyArray(), + content = "{\"name\":\"vitor\"}", + ) + store.insert(meta) + + val ftsRoot = root.resolve("idx/fts") + assertEquals(0, ftsRoot.listDirectoryEntries().size, "MetadataEvent is not SearchableEvent") + } + + @Test + fun `delete removes fts entries`() = + runBlocking { + val n = note("bitcoin nostr", ts = 100) + store.insert(n) + store.delete(n.id) + + val ftsRoot = root.resolve("idx/fts") + // Token directories may remain as empty husks. + for (tokenDir in ftsRoot.listDirectoryEntries()) { + assertEquals(0, tokenDir.listDirectoryEntries().size, "token entry leaked: $tokenDir") + } } - } // ------------------------------------------------------------------ // Search query semantics // ------------------------------------------------------------------ @Test - fun `single-token search returns the matching event`() { - val a = note("bitcoin is fun", ts = 1) - val b = note("nostr is also fun", ts = 2) - store.insert(a) - store.insert(b) + fun `single-token search returns the matching event`() = + runBlocking { + val a = note("bitcoin is fun", ts = 1) + val b = note("nostr is also fun", ts = 2) + store.insert(a) + store.insert(b) - val got = store.query(Filter(search = "bitcoin")) - assertEquals(listOf(a.id), got.map { it.id }) - } - - @Test - fun `multi-token search is AND across tokens`() { - val a = note("bitcoin only", ts = 1) - val b = note("nostr only", ts = 2) - val c = note("bitcoin and nostr", ts = 3) - store.insert(a) - store.insert(b) - store.insert(c) - - val got = store.query(Filter(search = "bitcoin nostr")) - assertEquals(listOf(c.id), got.map { it.id }, "AND semantics: only the doc with both tokens matches") - } - - @Test - fun `search results are ordered by createdAt DESC`() { - val older = note("bitcoin first", ts = 10) - val newer = note("bitcoin again", ts = 20) - store.insert(older) - store.insert(newer) - - val got = store.query(Filter(search = "bitcoin")) - assertEquals(listOf(newer.id, older.id), got.map { it.id }) - } - - @Test - fun `search respects limit`() { - repeat(5) { i -> store.insert(note("bitcoin doc $i", ts = i.toLong() + 1)) } - val got = store.query(Filter(search = "bitcoin", limit = 2)) - assertEquals(2, got.size) - } - - @Test - fun `search composes with kinds and authors via post-filter`() { - val match = note("bitcoin maximalism", ts = 5) - store.insert(match) - - val got = - store.query( - Filter(search = "bitcoin", kinds = listOf(1), authors = listOf(signer.pubKey)), - ) - assertEquals(listOf(match.id), got.map { it.id }) - - val miss = - store.query( - Filter(search = "bitcoin", kinds = listOf(2)), - ) - assertEquals(emptyList(), miss.map { it.id }) - } - - @Test - fun `search with no matching token returns empty`() { - store.insert(note("nostr only", ts = 1)) - assertEquals( - emptyList(), - store.query(Filter(search = "bitcoin")).map { it.id }, - ) - } - - @Test - fun `blank search string is ignored`() { - val a = note("anything", ts = 1) - store.insert(a) - // Blank search shouldn't drive by FTS — the planner falls through - // to all-kinds, and the event surfaces. - val got = store.query(Filter(search = " ")) - assertEquals(listOf(a.id), got.map { it.id }) - } - - @Test - fun `search survives reopen`() { - val n = note("persistent token", ts = 100) - store.insert(n) - store.close() - - val reopened = FsEventStore(root) - try { - val got = reopened.query(Filter(search = "persistent")) - assertEquals(listOf(n.id), got.map { it.id }) - } finally { - reopened.close() + val got = store.query(Filter(search = "bitcoin")) + assertEquals(listOf(a.id), got.map { it.id }) + } + + @Test + fun `multi-token search is AND across tokens`() = + runBlocking { + val a = note("bitcoin only", ts = 1) + val b = note("nostr only", ts = 2) + val c = note("bitcoin and nostr", ts = 3) + store.insert(a) + store.insert(b) + store.insert(c) + + val got = store.query(Filter(search = "bitcoin nostr")) + assertEquals(listOf(c.id), got.map { it.id }, "AND semantics: only the doc with both tokens matches") + } + + @Test + fun `search results are ordered by createdAt DESC`() = + runBlocking { + val older = note("bitcoin first", ts = 10) + val newer = note("bitcoin again", ts = 20) + store.insert(older) + store.insert(newer) + + val got = store.query(Filter(search = "bitcoin")) + assertEquals(listOf(newer.id, older.id), got.map { it.id }) + } + + @Test + fun `search respects limit`() = + runBlocking { + repeat(5) { i -> store.insert(note("bitcoin doc $i", ts = i.toLong() + 1)) } + val got = store.query(Filter(search = "bitcoin", limit = 2)) + assertEquals(2, got.size) + } + + @Test + fun `search composes with kinds and authors via post-filter`() = + runBlocking { + val match = note("bitcoin maximalism", ts = 5) + store.insert(match) + + val got = + store.query( + Filter(search = "bitcoin", kinds = listOf(1), authors = listOf(signer.pubKey)), + ) + assertEquals(listOf(match.id), got.map { it.id }) + + val miss = + store.query( + Filter(search = "bitcoin", kinds = listOf(2)), + ) + assertEquals(emptyList(), miss.map { it.id }) + } + + @Test + fun `search with no matching token returns empty`() = + runBlocking { + store.insert(note("nostr only", ts = 1)) + assertEquals( + emptyList(), + store.query(Filter(search = "bitcoin")).map { it.id }, + ) + } + + @Test + fun `blank search string is ignored`() = + runBlocking { + val a = note("anything", ts = 1) + store.insert(a) + // Blank search shouldn't drive by FTS — the planner falls through + // to all-kinds, and the event surfaces. + val got = store.query(Filter(search = " ")) + assertEquals(listOf(a.id), got.map { it.id }) + } + + @Test + fun `search survives reopen`() = + runBlocking { + val n = note("persistent token", ts = 100) + store.insert(n) + store.close() + + val reopened = FsEventStore(root) + try { + val got = reopened.query(Filter(search = "persistent")) + assertEquals(listOf(n.id), got.map { it.id }) + } finally { + reopened.close() + } } - } // ------------------------------------------------------------------ // Maintenance under replaceable / deletion / vanish // ------------------------------------------------------------------ @Test - fun `fts entry is unlinked when event is deleted`() { - val n = note("unique-token-zzz", ts = 1) - store.insert(n) - assertTrue(root.resolve("idx/fts/unique").exists()) - assertTrue(root.resolve("idx/fts/token").exists()) - assertTrue(root.resolve("idx/fts/zzz").exists()) + fun `fts entry is unlinked when event is deleted`() = + runBlocking { + val n = note("unique-token-zzz", ts = 1) + store.insert(n) + assertTrue(root.resolve("idx/fts/unique").exists()) + assertTrue(root.resolve("idx/fts/token").exists()) + assertTrue(root.resolve("idx/fts/zzz").exists()) - store.delete(n.id) - assertFalse( - root.resolve("idx/fts/zzz").let { it.exists() && it.listDirectoryEntries().isNotEmpty() }, - "zzz token entry should be unlinked", - ) + store.delete(n.id) + assertFalse( + root.resolve("idx/fts/zzz").let { it.exists() && it.listDirectoryEntries().isNotEmpty() }, + "zzz token entry should be unlinked", + ) - // And a search no longer finds it. - assertEquals(emptyList(), store.query(Filter(search = "zzz")).map { it.id }) - } + // And a search no longer finds it. + assertEquals(emptyList(), store.query(Filter(search = "zzz")).map { it.id }) + } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlotsTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlotsTest.kt index fd8eedeb0..8712fe8ce 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlotsTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsSlotsTest.kt @@ -26,6 +26,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -72,171 +73,181 @@ class FsSlotsTest { ) @Test - fun `newer replaceable evicts older`() { - val v1 = metadata("old", 100) - val v2 = metadata("new", 200) - store.insert(v1) - store.insert(v2) + fun `newer replaceable evicts older`() = + runBlocking { + val v1 = metadata("old", 100) + val v2 = metadata("new", 200) + store.insert(v1) + store.insert(v2) - // Only the newer survives a query by author. - val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) - assertEquals(listOf(v2.id), got.map { it.id }) + // Only the newer survives a query by author. + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) + assertEquals(listOf(v2.id), got.map { it.id }) - // The older canonical is gone. - assertFalse(store.hasCanonical(v1.id), "older canonical should be removed") - } + // The older canonical is gone. + assertFalse(store.hasCanonical(v1.id), "older canonical should be removed") + } @Test - fun `older replaceable is rejected when newer exists`() { - val newer = metadata("new", 200) - val older = metadata("old", 100) - store.insert(newer) - store.insert(older) + fun `older replaceable is rejected when newer exists`() = + runBlocking { + val newer = metadata("new", 200) + val older = metadata("old", 100) + store.insert(newer) + store.insert(older) - // Newer still wins. - val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) - assertEquals(listOf(newer.id), got.map { it.id }) + // Newer still wins. + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) + assertEquals(listOf(newer.id), got.map { it.id }) - // And the older was never persisted. - assertFalse(store.hasCanonical(older.id), "older should have been rejected") - } + // And the older was never persisted. + assertFalse(store.hasCanonical(older.id), "older should have been rejected") + } @Test - fun `equal timestamp replaceable resolves by lexical id (NIP-01)`() { - val a = metadata("a", 100) - val b = metadata("b", 100) - // NIP-01 tiebreaker: when createdAt ties, the lexically smaller - // id wins, regardless of insertion order. - val (winner, loser) = if (a.id < b.id) a to b else b to a + fun `equal timestamp replaceable resolves by lexical id (NIP-01)`() = + runBlocking { + val a = metadata("a", 100) + val b = metadata("b", 100) + // NIP-01 tiebreaker: when createdAt ties, the lexically smaller + // id wins, regardless of insertion order. + val (winner, loser) = if (a.id < b.id) a to b else b to a - // Loser inserted first, then winner — winner must replace. - store.insert(loser) - store.insert(winner) + // Loser inserted first, then winner — winner must replace. + store.insert(loser) + store.insert(winner) - val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) - assertEquals(1, got.size) - assertEquals(winner.id, got.single().id) - } + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) + assertEquals(1, got.size) + assertEquals(winner.id, got.single().id) + } @Test - fun `equal timestamp replaceable rejects higher id when winner already present`() { - val a = metadata("a", 100) - val b = metadata("b", 100) - val (winner, loser) = if (a.id < b.id) a to b else b to a + fun `equal timestamp replaceable rejects higher id when winner already present`() = + runBlocking { + val a = metadata("a", 100) + val b = metadata("b", 100) + val (winner, loser) = if (a.id < b.id) a to b else b to a - // Winner inserted first — loser must NOT take the slot. - store.insert(winner) - store.insert(loser) + // Winner inserted first — loser must NOT take the slot. + store.insert(winner) + store.insert(loser) - val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) - assertEquals(1, got.size) - assertEquals(winner.id, got.single().id) - } + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) + assertEquals(1, got.size) + assertEquals(winner.id, got.single().id) + } @Test - fun `replaceable slot file contains the current winner`() { - val v = metadata("only", 100) - store.insert(v) + fun `replaceable slot file contains the current winner`() = + runBlocking { + val v = metadata("only", 100) + store.insert(v) - val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") - assertTrue(slot.exists(), "slot must exist") - val parsed = Event.fromJson(slot.readText()) - assertEquals(v.id, parsed.id) - } + val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") + assertTrue(slot.exists(), "slot must exist") + val parsed = Event.fromJson(slot.readText()) + assertEquals(v.id, parsed.id) + } @Test - fun `replaceable slot survives canonical deletion via hardlink`() { - val v = metadata("x", 100) - store.insert(v) + fun `replaceable slot survives canonical deletion via hardlink`() = + runBlocking { + val v = metadata("x", 100) + store.insert(v) - // Simulate a user (or bug) removing the canonical file. - val canonical = - root - .resolve("events") - .resolve(v.id.substring(0, 2)) - .resolve(v.id.substring(2, 4)) - .resolve("${v.id}.json") - assertTrue(Files.deleteIfExists(canonical)) + // Simulate a user (or bug) removing the canonical file. + val canonical = + root + .resolve("events") + .resolve(v.id.substring(0, 2)) + .resolve(v.id.substring(2, 4)) + .resolve("${v.id}.json") + assertTrue(Files.deleteIfExists(canonical)) - val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") - assertTrue(slot.exists(), "slot should persist even if canonical is gone (hardlink to same inode)") - val parsed = Event.fromJson(slot.readText()) - assertEquals(v.id, parsed.id) - } + val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") + assertTrue(slot.exists(), "slot should persist even if canonical is gone (hardlink to same inode)") + val parsed = Event.fromJson(slot.readText()) + assertEquals(v.id, parsed.id) + } @Test - fun `eviction unlinks index hardlinks for the old winner`() { - val v1 = metadata("old", 100) - val v2 = metadata("new", 200) - store.insert(v1) - store.insert(v2) + fun `eviction unlinks index hardlinks for the old winner`() = + runBlocking { + val v1 = metadata("old", 100) + val v2 = metadata("new", 200) + store.insert(v1) + store.insert(v2) - // Author index should have exactly one entry — the winner. - val authorDir = root.resolve("idx/author/${signer.pubKey}") - val entries = - Files.list(authorDir).use { s -> - s.toList().map { it.fileName.toString() } - } - assertEquals(1, entries.size, "author index should only hold the winner") - assertTrue(entries.single().endsWith("-${v2.id}"), "author index entry must point at winner") - } - - @Test - fun `slot shortcut serves replaceable queries even when idx is wiped`() { - // Belt-and-suspenders for the planner shortcut: a query pinned to - // (kinds=[0], authors=[pk]) must hit the slot directly without - // touching idx/. Wipe idx/ to prove the shortcut isn't relying on - // it. - val v = metadata("p", 100) - store.insert(v) - java.nio.file.Files - .walk(root.resolve("idx")) - .use { s -> - s.sorted(Comparator.reverseOrder()).forEach { - java.nio.file.Files - .deleteIfExists(it) + // Author index should have exactly one entry — the winner. + val authorDir = root.resolve("idx/author/${signer.pubKey}") + val entries = + Files.list(authorDir).use { s -> + s.toList().map { it.fileName.toString() } } - } - - val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) - assertEquals(listOf(v.id), got.map { it.id }, "slot shortcut should serve from replaceable/, not idx/") - } + assertEquals(1, entries.size, "author index should only hold the winner") + assertTrue(entries.single().endsWith("-${v2.id}"), "author index entry must point at winner") + } @Test - fun `slot shortcut serves addressable queries when d-tag supplied`() { - val v = article("intro", "v", 10) - store.insert(v) - java.nio.file.Files - .walk(root.resolve("idx")) - .use { s -> - s.sorted(Comparator.reverseOrder()).forEach { - java.nio.file.Files - .deleteIfExists(it) + fun `slot shortcut serves replaceable queries even when idx is wiped`() = + runBlocking { + // Belt-and-suspenders for the planner shortcut: a query pinned to + // (kinds=[0], authors=[pk]) must hit the slot directly without + // touching idx/. Wipe idx/ to prove the shortcut isn't relying on + // it. + val v = metadata("p", 100) + store.insert(v) + java.nio.file.Files + .walk(root.resolve("idx")) + .use { s -> + s.sorted(Comparator.reverseOrder()).forEach { + java.nio.file.Files + .deleteIfExists(it) + } } - } - val got = - store.query( - Filter( - authors = listOf(signer.pubKey), - kinds = listOf(LongTextNoteEvent.KIND), - tags = mapOf("d" to listOf("intro")), - ), - ) - assertEquals(listOf(v.id), got.map { it.id }) - } + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(MetadataEvent.KIND))) + assertEquals(listOf(v.id), got.map { it.id }, "slot shortcut should serve from replaceable/, not idx/") + } @Test - fun `delete of current replaceable winner clears the slot`() { - val v = metadata("only", 100) - store.insert(v) + fun `slot shortcut serves addressable queries when d-tag supplied`() = + runBlocking { + val v = article("intro", "v", 10) + store.insert(v) + java.nio.file.Files + .walk(root.resolve("idx")) + .use { s -> + s.sorted(Comparator.reverseOrder()).forEach { + java.nio.file.Files + .deleteIfExists(it) + } + } - val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") - assertTrue(slot.exists()) + val got = + store.query( + Filter( + authors = listOf(signer.pubKey), + kinds = listOf(LongTextNoteEvent.KIND), + tags = mapOf("d" to listOf("intro")), + ), + ) + assertEquals(listOf(v.id), got.map { it.id }) + } - store.delete(v.id) - assertFalse(slot.exists(), "slot should be cleared when winner is deleted") - } + @Test + fun `delete of current replaceable winner clears the slot`() = + runBlocking { + val v = metadata("only", 100) + store.insert(v) + + val slot = root.resolve("replaceable/${MetadataEvent.KIND}/${signer.pubKey}.json") + assertTrue(slot.exists()) + + store.delete(v.id) + assertFalse(slot.exists(), "slot should be cleared when winner is deleted") + } // ------------------------------------------------------------------ // Addressable (kinds 30000-39999) @@ -255,100 +266,107 @@ class FsSlotsTest { ) @Test - fun `newer addressable evicts older for same d-tag`() { - val v1 = article("intro", "draft 1", 10) - val v2 = article("intro", "draft 2", 20) - store.insert(v1) - store.insert(v2) + fun `newer addressable evicts older for same d-tag`() = + runBlocking { + val v1 = article("intro", "draft 1", 10) + val v2 = article("intro", "draft 2", 20) + store.insert(v1) + store.insert(v2) - val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) - assertEquals(listOf(v2.id), got.map { it.id }) - assertFalse(store.hasCanonical(v1.id), "older draft canonical should be removed") - } + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) + assertEquals(listOf(v2.id), got.map { it.id }) + assertFalse(store.hasCanonical(v1.id), "older draft canonical should be removed") + } @Test - fun `addressable with different d-tags coexist`() { - val intro = article("intro", "hello", 10) - val about = article("about", "bio", 15) - store.insert(intro) - store.insert(about) + fun `addressable with different d-tags coexist`() = + runBlocking { + val intro = article("intro", "hello", 10) + val about = article("about", "bio", 15) + store.insert(intro) + store.insert(about) - val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) - assertEquals(setOf(intro.id, about.id), got.map { it.id }.toSet()) - } + val got = store.query(Filter(authors = listOf(signer.pubKey), kinds = listOf(LongTextNoteEvent.KIND))) + assertEquals(setOf(intro.id, about.id), got.map { it.id }.toSet()) + } @Test - fun `older addressable is rejected when newer exists`() { - val newer = article("slug", "new", 200) - val older = article("slug", "old", 100) - store.insert(newer) - store.insert(older) + fun `older addressable is rejected when newer exists`() = + runBlocking { + val newer = article("slug", "new", 200) + val older = article("slug", "old", 100) + store.insert(newer) + store.insert(older) - val got = store.query(Filter(authors = listOf(signer.pubKey))) - assertEquals(listOf(newer.id), got.map { it.id }) - } + val got = store.query(Filter(authors = listOf(signer.pubKey))) + assertEquals(listOf(newer.id), got.map { it.id }) + } @Test - fun `addressable slot file contains the current winner`() { - val v = article("intro", "hello", 10) - store.insert(v) + fun `addressable slot file contains the current winner`() = + runBlocking { + val v = article("intro", "hello", 10) + store.insert(v) - val dHash = FsLayout.sha256Hex("intro") - val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") - assertTrue(slot.exists()) - val parsed = Event.fromJson(slot.readText()) - assertEquals(v.id, parsed.id) - } + val dHash = FsLayout.sha256Hex("intro") + val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") + assertTrue(slot.exists()) + val parsed = Event.fromJson(slot.readText()) + assertEquals(v.id, parsed.id) + } @Test - fun `empty d-tag gets its own slot`() { - val v = article("", "homepage", 1) - store.insert(v) + fun `empty d-tag gets its own slot`() = + runBlocking { + val v = article("", "homepage", 1) + store.insert(v) - val dHash = FsLayout.sha256Hex("") - val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") - assertTrue(slot.exists()) - } + val dHash = FsLayout.sha256Hex("") + val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") + assertTrue(slot.exists()) + } @Test - fun `delete of current addressable winner clears the slot`() { - val v = article("intro", "hello", 10) - store.insert(v) - val dHash = FsLayout.sha256Hex("intro") - val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") - assertTrue(slot.exists()) + fun `delete of current addressable winner clears the slot`() = + runBlocking { + val v = article("intro", "hello", 10) + store.insert(v) + val dHash = FsLayout.sha256Hex("intro") + val slot = root.resolve("addressable/${LongTextNoteEvent.KIND}/${signer.pubKey}/$dHash.json") + assertTrue(slot.exists()) - store.delete(v.id) - assertFalse(slot.exists()) - } + store.delete(v.id) + assertFalse(slot.exists()) + } // ------------------------------------------------------------------ // Non-replaceable events: no slot involvement // ------------------------------------------------------------------ @Test - fun `regular text note has no slot`() { - val note = - signer.sign( - createdAt = 1, - kind = 1, - tags = emptyArray(), - content = "plain", - ) - store.insert(note) + fun `regular text note has no slot`() = + runBlocking { + val note = + signer.sign( + createdAt = 1, + kind = 1, + tags = emptyArray(), + content = "plain", + ) + store.insert(note) - // No entries under replaceable/ or addressable/ — only the scaffolded dirs exist. - val replaceableDir = root.resolve("replaceable") - val addressableDir = root.resolve("addressable") - assertEquals( - 0, - Files.walk(replaceableDir).use { s -> s.filter { Files.isRegularFile(it) }.count() }, - ) - assertEquals( - 0, - Files.walk(addressableDir).use { s -> s.filter { Files.isRegularFile(it) }.count() }, - ) - } + // No entries under replaceable/ or addressable/ — only the scaffolded dirs exist. + val replaceableDir = root.resolve("replaceable") + val addressableDir = root.resolve("addressable") + assertEquals( + 0, + Files.walk(replaceableDir).use { s -> s.filter { Files.isRegularFile(it) }.count() }, + ) + assertEquals( + 0, + Files.walk(addressableDir).use { s -> s.filter { Files.isRegularFile(it) }.count() }, + ) + } // helper — check canonical existence private fun FsEventStore.hasCanonical(id: String): Boolean { diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsVanishTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsVanishTest.kt index 0e824a865..18703ed43 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsVanishTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsVanishTest.kt @@ -27,6 +27,7 @@ import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.runBlocking import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.exists @@ -81,171 +82,182 @@ class FsVanishTest { // ------------------------------------------------------------------ @Test - fun `vanish for this relay cascades older events from the same author`() { - val n1 = note("a", 10) - val n2 = note("b", 20) - val n3 = note("c", 30) // same ts as vanish — survives because cascade uses strict < - store.insert(n1) - store.insert(n2) - store.insert(n3) + fun `vanish for this relay cascades older events from the same author`() = + runBlocking { + val n1 = note("a", 10) + val n2 = note("b", 20) + val n3 = note("c", 30) // same ts as vanish — survives because cascade uses strict < + store.insert(n1) + store.insert(n2) + store.insert(n3) - val v = vanish(ts = 30) - store.insert(v) + val v = vanish(ts = 30) + store.insert(v) - assertFalse(store.hasCanonical(n1.id), "n1 should be cascade-deleted") - assertFalse(store.hasCanonical(n2.id), "n2 should be cascade-deleted") - assertTrue(store.hasCanonical(n3.id), "n3 (createdAt == vanish.createdAt) survives") - assertTrue(store.hasCanonical(v.id)) - } - - @Test - fun `vanish for a different relay does NOT cascade`() { - val n = note("x", 10) - store.insert(n) - - val v = vanish(ts = 20, relayUrl = "wss://elsewhere.example") - store.insert(v) - - assertTrue(store.hasCanonical(n.id), "vanish scoped to another relay must not cascade") - // The kind-62 event itself is still persisted (it's just a normal event). - assertTrue(store.hasCanonical(v.id)) - // No vanish tombstone installed. - val tombDir = root.resolve("tombstones/vanish") - if (tombDir.exists()) { - assertEquals(0, Files.list(tombDir).use { it.toList() }.size) + assertFalse(store.hasCanonical(n1.id), "n1 should be cascade-deleted") + assertFalse(store.hasCanonical(n2.id), "n2 should be cascade-deleted") + assertTrue(store.hasCanonical(n3.id), "n3 (createdAt == vanish.createdAt) survives") + assertTrue(store.hasCanonical(v.id)) } - } @Test - fun `vanishFromEverywhere always cascades regardless of relay`() { - val n = note("x", 10) - store.insert(n) + fun `vanish for a different relay does NOT cascade`() = + runBlocking { + val n = note("x", 10) + store.insert(n) - val v = vanishEverywhere(ts = 20) - store.insert(v) + val v = vanish(ts = 20, relayUrl = "wss://elsewhere.example") + store.insert(v) - assertFalse(store.hasCanonical(n.id)) - } + assertTrue(store.hasCanonical(n.id), "vanish scoped to another relay must not cascade") + // The kind-62 event itself is still persisted (it's just a normal event). + assertTrue(store.hasCanonical(v.id)) + // No vanish tombstone installed. + val tombDir = root.resolve("tombstones/vanish") + if (tombDir.exists()) { + assertEquals(0, Files.list(tombDir).use { it.toList() }.size) + } + } + + @Test + fun `vanishFromEverywhere always cascades regardless of relay`() = + runBlocking { + val n = note("x", 10) + store.insert(n) + + val v = vanishEverywhere(ts = 20) + store.insert(v) + + assertFalse(store.hasCanonical(n.id)) + } // ------------------------------------------------------------------ // Block re-insert // ------------------------------------------------------------------ @Test - fun `events older than vanish are blocked from re-insertion`() { - val n = note("a", 10) - store.insert(n) + fun `events older than vanish are blocked from re-insertion`() = + runBlocking { + val n = note("a", 10) + store.insert(n) - val v = vanish(ts = 50) - store.insert(v) + val v = vanish(ts = 50) + store.insert(v) - // Re-insert blocked. - store.insert(n) - assertFalse(store.hasCanonical(n.id)) + // Re-insert blocked. + store.insert(n) + assertFalse(store.hasCanonical(n.id)) - // A brand-new older event by the same author also blocked. - val older = note("older", 5) - store.insert(older) - assertFalse(store.hasCanonical(older.id)) - } + // A brand-new older event by the same author also blocked. + val older = note("older", 5) + store.insert(older) + assertFalse(store.hasCanonical(older.id)) + } @Test - fun `events at vanish ts are blocked, parity with SQLite`() { - val v = vanish(ts = 50) - store.insert(v) + fun `events at vanish ts are blocked, parity with SQLite`() = + runBlocking { + val v = vanish(ts = 50) + store.insert(v) - val equal = note("equal", 50) - store.insert(equal) - assertFalse(store.hasCanonical(equal.id), "createdAt == vanish.createdAt should be blocked") - } + val equal = note("equal", 50) + store.insert(equal) + assertFalse(store.hasCanonical(equal.id), "createdAt == vanish.createdAt should be blocked") + } @Test - fun `events newer than vanish still pass`() { - val v = vanish(ts = 50) - store.insert(v) + fun `events newer than vanish still pass`() = + runBlocking { + val v = vanish(ts = 50) + store.insert(v) - val newer = note("newer", 100) - store.insert(newer) - assertTrue(store.hasCanonical(newer.id)) - } + val newer = note("newer", 100) + store.insert(newer) + assertTrue(store.hasCanonical(newer.id)) + } @Test - fun `another author is unaffected by my vanish`() { - val mine = note("mine", 10) - store.insert(mine) - val theirs = note("theirs", 5, s = otherSigner) - store.insert(theirs) + fun `another author is unaffected by my vanish`() = + runBlocking { + val mine = note("mine", 10) + store.insert(mine) + val theirs = note("theirs", 5, s = otherSigner) + store.insert(theirs) - val v = vanish(ts = 50) - store.insert(v) + val v = vanish(ts = 50) + store.insert(v) - assertFalse(store.hasCanonical(mine.id), "my old event cascade-deleted") - assertTrue(store.hasCanonical(theirs.id), "other author's event is unaffected") - } + assertFalse(store.hasCanonical(mine.id), "my old event cascade-deleted") + assertTrue(store.hasCanonical(theirs.id), "other author's event is unaffected") + } // ------------------------------------------------------------------ // Multiple vanish requests — strongest cutoff wins // ------------------------------------------------------------------ @Test - fun `later vanish raises the cutoff`() { - val n100 = note("at-100", 100) - store.insert(n100) - store.insert(vanish(ts = 50)) - // n100 still around because 100 > 50. - assertTrue(store.hasCanonical(n100.id)) + fun `later vanish raises the cutoff`() = + runBlocking { + val n100 = note("at-100", 100) + store.insert(n100) + store.insert(vanish(ts = 50)) + // n100 still around because 100 > 50. + assertTrue(store.hasCanonical(n100.id)) - // Stronger vanish at ts=200 cascades it. - store.insert(vanish(ts = 200)) - assertFalse(store.hasCanonical(n100.id)) + // Stronger vanish at ts=200 cascades it. + store.insert(vanish(ts = 200)) + assertFalse(store.hasCanonical(n100.id)) - // And new events at ts=150 are now blocked. - val mid = note("mid", 150) - store.insert(mid) - assertFalse(store.hasCanonical(mid.id)) - } + // And new events at ts=150 are now blocked. + val mid = note("mid", 150) + store.insert(mid) + assertFalse(store.hasCanonical(mid.id)) + } @Test - fun `earlier vanish does not lower a stronger cutoff`() { - store.insert(vanish(ts = 200)) - store.insert(vanish(ts = 50)) // older — should be a no-op for the tombstone + fun `earlier vanish does not lower a stronger cutoff`() = + runBlocking { + store.insert(vanish(ts = 200)) + store.insert(vanish(ts = 50)) // older — should be a no-op for the tombstone - val mid = note("mid", 150) - store.insert(mid) - assertFalse(store.hasCanonical(mid.id), "stronger cutoff stays at 200") - } + val mid = note("mid", 150) + store.insert(mid) + assertFalse(store.hasCanonical(mid.id), "stronger cutoff stays at 200") + } // ------------------------------------------------------------------ // Tombstone is a hardlink to the kind-62 event // ------------------------------------------------------------------ @Test - fun `vanish tombstone shares an inode with the kind-62 event`() { - val v = vanish(ts = 30) - store.insert(v) + fun `vanish tombstone shares an inode with the kind-62 event`() = + runBlocking { + val v = vanish(ts = 30) + store.insert(v) - val tombDir = root.resolve("tombstones/vanish") - val entries = Files.list(tombDir).use { it.toList() } - assertEquals(1, entries.size) + val tombDir = root.resolve("tombstones/vanish") + val entries = Files.list(tombDir).use { it.toList() } + assertEquals(1, entries.size) - val canonical = root.resolve("events/${v.id.substring(0, 2)}/${v.id.substring(2, 4)}/${v.id}.json") - val tombKey = Files.readAttributes(entries.single(), java.nio.file.attribute.BasicFileAttributes::class.java).fileKey() - val canKey = Files.readAttributes(canonical, java.nio.file.attribute.BasicFileAttributes::class.java).fileKey() - assertEquals(canKey, tombKey, "vanish tombstone should be a hardlink to the kind-62 canonical") - } + val canonical = root.resolve("events/${v.id.substring(0, 2)}/${v.id.substring(2, 4)}/${v.id}.json") + val tombKey = Files.readAttributes(entries.single(), java.nio.file.attribute.BasicFileAttributes::class.java).fileKey() + val canKey = Files.readAttributes(canonical, java.nio.file.attribute.BasicFileAttributes::class.java).fileKey() + assertEquals(canKey, tombKey, "vanish tombstone should be a hardlink to the kind-62 canonical") + } // ------------------------------------------------------------------ // Vanish event itself remains queryable // ------------------------------------------------------------------ @Test - fun `vanish event itself is indexed and queryable`() { - val v = vanish(ts = 30) - store.insert(v) + fun `vanish event itself is indexed and queryable`() = + runBlocking { + val v = vanish(ts = 30) + store.insert(v) - val byKind = store.query(Filter(kinds = listOf(RequestToVanishEvent.KIND))) - assertEquals(listOf(v.id), byKind.map { it.id }) - } + val byKind = store.query(Filter(kinds = listOf(RequestToVanishEvent.KIND))) + assertEquals(listOf(v.id), byKind.map { it.id }) + } private fun FsEventStore.hasCanonical(id: String): Boolean { val p = diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ParallelInsertTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ParallelInsertTest.kt new file mode 100644 index 000000000..eeff076f9 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ParallelInsertTest.kt @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.deleteIfExists +import kotlin.io.path.exists +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Stress test for the SQLite connection pool. Pre-pool, two coroutines + * inserting at the same time would race on the shared `SQLiteConnection` + * (`androidx.sqlite` connections aren't thread-safe) and crash with + * either `SQLITE_ERROR: cannot start a transaction within a transaction` + * or a corrupted prepared statement (`SQLITE_MISUSE`). + * + * With [SQLiteConnectionPool] writes serialise behind a coroutine `Mutex` + * and reads run in parallel against a fixed pool of reader connections, + * matching what Room does. The test launches a fan-out of inserts and + * concurrent reads, then asserts every inserted event is visible and the + * count is exact. + */ +class ParallelInsertTest { + private val signer = NostrSignerSync() + private lateinit var dbFile: Path + private lateinit var store: EventStore + + @BeforeTest + fun setup() { + Secp256k1Instance + // Use a real file so the pool can hand out independent reader + // connections — :memory: would make every connection a separate DB. + dbFile = Files.createTempFile("parallel-insert-", ".db") + // Driver expects to open the file itself; ensure the placeholder + // is gone so SQLite can create a fresh DB. + Files.deleteIfExists(dbFile) + store = EventStore(dbName = dbFile.toAbsolutePath().toString(), relay = null) + } + + @AfterTest + fun tearDown() { + store.close() + // SQLite leaves -wal / -shm sidecars next to the main file under WAL. + listOf("", "-wal", "-shm", "-journal").forEach { suffix -> + Path.of(dbFile.toString() + suffix).deleteIfExists() + } + } + + @Test + fun `parallel inserts on N coroutines all succeed`() = + runBlocking { + val perCoroutine = 200 + val coroutines = 8 + val total = perCoroutine * coroutines + + val events = + (0 until total).map { i -> + signer.sign(TextNoteEvent.build("p$i", createdAt = i.toLong() + 1)) + } + + // Fan out inserts across `coroutines` workers on the IO + // dispatcher (multi-thread). Without the pool's writer mutex + // these all race on a single SQLiteConnection and crash. + coroutineScope { + events.chunked(perCoroutine).forEach { chunk -> + launch(Dispatchers.IO) { + for (e in chunk) store.insert(e) + } + } + } + + assertEquals(total, store.count(Filter()), "every insert must be visible") + + val byId = store.query(Filter()).associateBy { it.id } + for (e in events) { + assertTrue(byId.containsKey(e.id), "missing event ${e.id.take(8)}") + } + } + + @Test + fun `parallel reads run alongside writes without crashing`() = + runBlocking { + val writes = 500 + + val events = + (0 until writes).map { i -> + signer.sign(TextNoteEvent.build("rw$i", createdAt = i.toLong() + 1)) + } + + coroutineScope { + // Writer feed. + launch(Dispatchers.IO) { + for (e in events) store.insert(e) + } + // Multiple reader fans-out: count() and query() running + // continuously while inserts are still in flight. Asserts + // none of these crash with SQLITE_MISUSE. + val readers = + List(4) { + async(Dispatchers.IO) { + var lastSeen = 0 + repeat(100) { + val n = store.count(Filter()) + assertTrue(n in 0..writes) + if (n > lastSeen) lastSeen = n + } + lastSeen + } + } + readers.awaitAll() + } + + assertEquals(writes, store.count(Filter())) + } + + @Test + fun `parallel transaction batches all commit`() = + runBlocking { + val batches = 8 + val perBatch = 50 + val total = batches * perBatch + + val events = + (0 until total).map { i -> + signer.sign(TextNoteEvent.build("t$i", createdAt = i.toLong() + 1)) + } + + // Each coroutine wraps its slice in store.transaction { ... }, + // exercising the writer mutex around BEGIN/COMMIT pairs. + coroutineScope { + events.chunked(perBatch).forEach { chunk -> + launch(Dispatchers.IO) { + store.transaction { + for (e in chunk) insert(e) + } + } + } + } + + assertEquals(total, store.count(Filter())) + } + + @Test + fun `pool with file-backed db survives reopen`() = + runBlocking { + // Smoke test that the pool migration runs idempotently when + // a writer connection is reopened against an existing DB. + val first = signer.sign(TextNoteEvent.build("first", createdAt = 1)) + store.insert(first) + store.close() + + val reopened = EventStore(dbName = dbFile.toAbsolutePath().toString(), relay = null) + try { + assertTrue(dbFile.exists()) + val got = reopened.query(Filter(ids = listOf(first.id))) + assertEquals(listOf(first.id), got.map { it.id }) + + // And then more parallel inserts still work on the + // reopened pool. + val moreCount = 20 + val more = (0 until moreCount).map { signer.sign(TextNoteEvent.build("m$it", createdAt = it.toLong() + 100)) } + coroutineScope { + more.forEach { e -> + launch(Dispatchers.IO) { reopened.insert(e) } + } + } + assertEquals(1 + moreCount, reopened.count(Filter())) + } finally { + reopened.close() + } + } +}