From d370784f3f526cdc5d0f5267d8ee193f520daffe Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 19:02:40 +0000 Subject: [PATCH 01/24] feat(quartz): reactive EventStoreProjection over SQLiteEventStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a stateful observer that turns a Filter into a StateFlow>>: - A TEMP table + AFTER DELETE trigger on event_headers logs OLD.id for every row that leaves the store, regardless of cause (replaceable / addressable supersession, NIP-09 deletion, NIP-62 vanish, NIP-40 expiration sweep, manual delete, clearDB). - SQLiteEventStore drains the log around each writer unit of work and emits a StoreChange(inserted, removedIds) on a SharedFlow. - EventStoreProjection seeds itself from the store, then maintains stable MutableStateFlow handles keyed by (kind:pubkey:dtag) for addressables and by id otherwise. Addressable updates mutate the handle's value in place — list reference stable; insert / remove rebuilds the list reference. - 11 new tests cover seed, insert, replaceable update, addressable update, NIP-09 deletion, NIP-62 vanish, NIP-40 expiration, manual delete, limit enforcement, non-matching insert, and close-cancels-listener. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../nip01Core/store/sqlite/ChangeLogModule.kt | 98 +++++ .../nip01Core/store/sqlite/EventStore.kt | 21 ++ .../store/sqlite/EventStoreProjection.kt | 247 +++++++++++++ .../store/sqlite/SQLiteConnectionPool.kt | 4 +- .../store/sqlite/SQLiteEventStore.kt | 76 +++- .../nip01Core/store/sqlite/StoreChange.kt | 57 +++ .../store/sqlite/EventStoreProjectionTest.kt | 347 ++++++++++++++++++ 7 files changed, 842 insertions(+), 8 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ChangeLogModule.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjection.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StoreChange.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjectionTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ChangeLogModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ChangeLogModule.kt new file mode 100644 index 000000000..73d34cd95 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ChangeLogModule.kt @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import androidx.sqlite.SQLiteConnection +import com.vitorpamplona.quartz.nip01Core.core.HexKey + +/** + * Per-connection scratch space that records the id of every row that + * leaves `event_headers`, regardless of why it left. + * + * Why TEMP: the table and trigger are connection-scoped, so they only + * exist on the writer connection where they're installed. Readers never + * see them and never accumulate rows. The data also lives only for the + * lifetime of the connection — exactly the right scope for "ids removed + * since the last drain." + * + * The trigger fires after every delete on `event_headers`, including: + * - the supersession triggers in [ReplaceableModule] / [AddressableModule], + * - the cascade from `event_vanish` in [RightToVanishModule], + * - the explicit deletes in [DeletionRequestModule], + * - [ExpirationModule.deleteExpiredEvents], + * - manual `delete(filter)` / `delete(id)`, + * - `clearDB()`. + * + * The [SQLiteEventStore] drains this log around each unit of work and + * publishes the resulting ids in a [StoreChange]. + */ +class ChangeLogModule { + fun installOnWriter(db: SQLiteConnection) { + db.execSQL( + """ + CREATE TEMP TABLE IF NOT EXISTS event_change_log ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL + ) + """.trimIndent(), + ) + + db.execSQL( + """ + CREATE TEMP TRIGGER IF NOT EXISTS event_change_log_on_delete + AFTER DELETE ON event_headers + FOR EACH ROW + BEGIN + INSERT INTO event_change_log (id) VALUES (OLD.id); + END + """.trimIndent(), + ) + } + + /** + * Reads every id currently logged and clears the log. + * + * Must be called on the writer connection that owns the temp table. + * Callers are expected to be inside the writer mutex (drain happens + * inside or right after the `useWriter { ... }` block); a single + * read-and-clear pair is therefore atomic from the writer's view. + */ + fun drain(db: SQLiteConnection): List { + val ids = ArrayList() + db.prepare("SELECT id FROM event_change_log ORDER BY seq").use { stmt -> + while (stmt.step()) { + ids.add(stmt.getText(0)) + } + } + if (ids.isNotEmpty()) { + db.execSQL("DELETE FROM event_change_log") + } + return ids + } + + /** + * Drops everything from the log without returning it. Used after a + * rollback so the next successful unit of work starts clean. + */ + fun reset(db: SQLiteConnection) { + db.execSQL("DELETE FROM event_change_log") + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index 7fb287555..f17be61ee 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -26,6 +26,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import kotlinx.coroutines.CoroutineScope class EventStore( dbName: String? = "events.db", @@ -66,5 +67,25 @@ class EventStore( override suspend fun deleteExpiredEvents() = store.deleteExpiredEvents() + /** + * Stream of mutations committed to the store. See [SQLiteEventStore.changes]. + */ + val changes get() = store.changes + + /** + * Open a reactive [EventStoreProjection] over the store for + * [filters]. The projection runs inside [scope]; cancel the scope + * (or call [EventStoreProjection.close]) to release it. + */ + fun observe( + filters: List, + scope: CoroutineScope, + ): EventStoreProjection = EventStoreProjection(store, filters, scope) + + fun observe( + filter: Filter, + scope: CoroutineScope, + ): EventStoreProjection = observe(listOf(filter), scope) + override fun close() = store.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjection.kt new file mode 100644 index 000000000..f655d6dfd --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjection.kt @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.yield + +/** + * A reactive projection over the [SQLiteEventStore] for a fixed set of + * [filters]. Each visible event is wrapped in a [MutableStateFlow] so + * the UI can collect three different kinds of change with the right + * granularity: + * + * - **Membership** (events arriving or leaving) re-emits a brand new + * [List] from [items]. The list reference is stable while membership + * is unchanged. + * - **In-place addressable update** (a new version of the same + * `kind:pubkey:dtag` arrives) updates the existing handle's + * [MutableStateFlow.value] without touching the list. Only collectors + * of that one handle re-render. The list ordering is *not* reshuffled + * when the new version has a later `created_at` — each slot remembers + * the sort key it was inserted with, so addressable updates feel like + * pure value mutations. + * - **Removal** (NIP-09 deletion, NIP-62 vanish, NIP-40 expiration, or + * a non-addressable event being explicitly deleted) drops the handle + * from the list. + * + * The seed is materialised by running the filters against the store + * once at start. After that, [items] is driven entirely by + * [SQLiteEventStore.changes]; the database is not re-queried on every + * mutation. + * + * Limit handling matches the existing in-memory observables in + * `commons/observables`: the initial query honours the filter `limit`, + * and we trim the list to the same cap when an insert pushes it over. + * We do **not** refill from the DB after a deletion — if a deletion + * leaves you under the limit, you stay under the limit until something + * new arrives. That tradeoff keeps the projection allocation-free per + * mutation and matches what callers were already getting from + * `LocalCache.observeEvents`. + * + * Lifecycle: the projection runs a single coroutine in [scope]. Cancel + * the scope (or call [close]) when the screen using the projection + * goes away. There is no shared state between projections; each one + * keeps its own indexes. + */ +class EventStoreProjection( + private val store: SQLiteEventStore, + private val filters: List, + scope: CoroutineScope, +) : AutoCloseable { + private val _items = MutableStateFlow>>(emptyList()) + val items: StateFlow>> = _items.asStateFlow() + + /** Slots keyed by the *current* event id. Re-keyed when an addressable handle takes a new version. */ + private val byId = HashMap>() + + /** Slots keyed by `kind:pubkey:dtag` for in-place addressable updates. */ + private val byAddress = HashMap>() + + /** + * Sorted view of the same slots. The comparator uses each slot's + * frozen sort key (the seed event's `created_at` + `id`), so an + * addressable update never moves a slot inside this set. + */ + private val ordered = sortedSetOf(slotComparator()) + + private val limit: Int? = filters.mapNotNull { it.limit }.maxOrNull() + + /** Set when the seed has been written to [items], so callers can suspend until the projection is hot. */ + val ready: CompletableDeferred = CompletableDeferred() + + private val job: Job = + scope.launch { + seed() + ready.complete(Unit) + store.changes.collect { change -> apply(change) } + } + + private suspend fun seed() { + val initial = store.query(filters) + for (event in initial) { + insertNew(event) + // Cooperate with cancellation on very large seeds. + yield() + } + publish() + } + + private fun apply(change: StoreChange) { + var changed = false + + // Removals first. A replaceable / addressable supersession + // arrives as `inserted = [new]` plus `removedIds = [oldId]`. + // When the new event is addressable and an existing slot + // already maps that address, [handleInsert] will rekey + // `byId` from the old id to the new id before this loop sees + // the old id — so the lookup here is a no-op for that case + // and the slot stays in place. For non-addressable + // replaceables (kinds 0/3/10000-19999) the old event has a + // different id, no address index, and we genuinely drop it. + for (event in change.inserted) { + if (handleInsert(event)) changed = true + } + for (id in change.removedIds) { + if (handleRemove(id)) changed = true + } + + if (changed) publish() + } + + @Suppress("UNCHECKED_CAST") + private fun handleInsert(event: Event): Boolean { + if (filters.none { it.match(event) }) return false + + if (event is AddressableEvent) { + val key = event.addressTag() + val existing = byAddress[key] + if (existing != null) { + // Same address, new version. Rekey byId from the + // previous event id to the new one and update the + // handle's value in place — list reference does not + // change, only the handle's collectors re-render. + val previousId = existing.flow.value.id + if (previousId != event.id) { + byId.remove(previousId) + byId[event.id] = existing + } + existing.flow.value = event as T + return false + } + } else if (byId.containsKey(event.id)) { + return false + } + + insertNew(event) + return true + } + + private fun handleRemove(id: HexKey): Boolean { + val slot = byId.remove(id) ?: return false + ordered.remove(slot) + val ev = slot.flow.value + if (ev is AddressableEvent) { + val addr = ev.addressTag() + // Only clear the address index if this slot still owns it. + if (byAddress[addr] === slot) byAddress.remove(addr) + } + return true + } + + @Suppress("UNCHECKED_CAST") + private fun insertNew(event: Event) { + val slot = Slot(event as T) + byId[event.id] = slot + if (event is AddressableEvent) byAddress[event.addressTag()] = slot + ordered.add(slot) + + val cap = limit ?: return + while (ordered.size > cap) { + val tail = ordered.last() + ordered.remove(tail) + val tailEvent = tail.flow.value + byId.remove(tailEvent.id) + if (tailEvent is AddressableEvent) { + val addr = tailEvent.addressTag() + if (byAddress[addr] === tail) byAddress.remove(addr) + } + } + } + + private fun publish() { + _items.value = ordered.map { it.flow } + } + + /** + * Stop tracking changes and clear internal state. Idempotent. The + * scope passed to the constructor keeps running; only this + * projection's collector job is cancelled. + */ + override fun close() { + job.cancel() + ordered.clear() + byId.clear() + byAddress.clear() + _items.value = emptyList() + } + + /** + * Internal slot. Each event added to the projection lives inside + * one of these for as long as it survives. The sort key is frozen + * at construction time — addressable in-place updates rewrite + * `flow.value` but never the sort key, so the ordering inside + * [ordered] is stable across updates. + */ + private class Slot( + initial: T, + ) { + val sortCreatedAt: Long = initial.createdAt + val sortId: HexKey = initial.id + val flow: MutableStateFlow = MutableStateFlow(initial) + } + + companion object { + /** + * created_at DESC, id ASC. The keys are snapshots taken at + * insertion time, so the ordering of a slot never changes + * after it joins the set. Distinct events have distinct ids, + * so no third-key tiebreak is needed. + */ + private fun slotComparator(): Comparator> = + Comparator { a, b -> + if (a === b) return@Comparator 0 + val byTime = b.sortCreatedAt.compareTo(a.sortCreatedAt) + if (byTime != 0) return@Comparator byTime + a.sortId.compareTo(b.sortId) + } + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt index 5ca19d985..2e4125b90 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt @@ -107,7 +107,7 @@ class SQLiteConnectionPool( * suspend until the lock is released. Cancellation-aware via the * coroutine [Mutex]. */ - suspend fun useWriter(block: (SQLiteConnection) -> T): T = + suspend fun useWriter(block: suspend (SQLiteConnection) -> T): T = writerMutex.withLock { block(writer) } @@ -118,7 +118,7 @@ class SQLiteConnectionPool( * (WAL). With an in-memory DB this falls back to the writer mutex * because each `:memory:` connection would be a separate database. */ - suspend fun useReader(block: (SQLiteConnection) -> T): T { + suspend fun useReader(block: suspend (SQLiteConnection) -> T): T { val ch = readerChannel ?: return writerMutex.withLock { block(writer) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 2da5cf49c..83c95d531 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -34,6 +34,10 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip40Expiration.isExpired import com.vitorpamplona.quartz.utils.EventFactory +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.asSharedFlow class SQLiteEventStore( val driver: SQLiteDriver = BundledSQLiteDriver(), @@ -62,6 +66,22 @@ class SQLiteEventStore( val deletionModule = DeletionRequestModule(seedModule::hasher) val expirationModule = ExpirationModule() val rightToVanishModule = RightToVanishModule(seedModule::hasher) + val changeLogModule = ChangeLogModule() + + /** + * Stream of mutations committed to the store. One [StoreChange] per + * successful unit of work; rejected inserts and rolled-back + * transactions are not emitted. Subscribers see updates in commit + * order. See [EventStoreProjection] for a high-level reactive list + * that consumes this stream. + */ + private val _changes = + MutableSharedFlow( + replay = 0, + extraBufferCapacity = 256, + onBufferOverflow = BufferOverflow.SUSPEND, + ) + val changes: SharedFlow = _changes.asSharedFlow() val queryBuilder = QueryBuilder( @@ -119,6 +139,12 @@ class SQLiteEventStore( setUserVersion(this, DATABASE_VERSION) } } + + // After the schema is in place (fresh or already-current + // DBs both reach this point), wire the change log onto + // the writer connection. TEMP table + TEMP trigger are + // connection-scoped, so this runs on the writer only. + changeLogModule.installOnWriter(db) }, ) } @@ -160,10 +186,23 @@ class SQLiteEventStore( } } - suspend fun clearDB() = + suspend fun clearDB() { pool.useWriter { db -> modules.reversed().forEach { it.deleteAll(db) } + // The deleteAll cascade fires the change-log trigger for + // every removed event_header row. Drain and publish so + // open projections drop everything they were holding. + publishChange(emptyList(), changeLogModule.drain(db)) } + } + + private suspend fun publishChange( + inserted: List, + removedIds: List, + ) { + if (inserted.isEmpty() && removedIds.isEmpty()) return + _changes.emit(StoreChange(inserted, removedIds)) + } suspend fun vacuum() = pool.useWriter { db -> @@ -198,27 +237,38 @@ class SQLiteEventStore( db.transaction { innerInsertEvent(event, this) } + // The transaction either committed or threw. On commit, the + // change log holds ids superseded by replaceable / addressable + // triggers and any NIP-09 / NIP-62 cascades fired by this + // event's content. On rollback the temp table was rolled back + // with the transaction, so drain returns empty. + publishChange(listOf(event), changeLogModule.drain(db)) } } inner class Transaction( val db: SQLiteConnection, ) : IEventStore.ITransaction { + val accepted = ArrayList() + override fun insert(event: Event) { if (event.isExpired()) throw SQLiteException("blocked: Cannot insert an expired event") if (event.kind.isEphemeral()) return innerInsertEvent(event, db) + accepted.add(event) } } suspend fun transaction(body: Transaction.() -> Unit) { pool.useWriter { db -> + val txn = Transaction(db) db.transaction { - with(Transaction(this)) { + with(txn) { body() } } + publishChange(txn.accepted, changeLogModule.drain(db)) } } @@ -258,17 +308,31 @@ class SQLiteEventStore( suspend fun count(filters: List): Int = pool.useReader { queryBuilder.count(filters, it) } - suspend fun delete(filter: Filter) = pool.useWriter { queryBuilder.delete(filter, it) } + suspend fun delete(filter: Filter) = + pool.useWriter { db -> + queryBuilder.delete(filter, db) + publishChange(emptyList(), changeLogModule.drain(db)) + } - suspend fun delete(filters: List) = pool.useWriter { queryBuilder.delete(filters, it) } + suspend fun delete(filters: List) = + pool.useWriter { db -> + queryBuilder.delete(filters, db) + publishChange(emptyList(), changeLogModule.drain(db)) + } suspend fun delete(id: HexKey): Int = pool.useWriter { db -> db.execSQL("DELETE FROM event_headers WHERE id = ?", arrayOf(id)) - db.changes() + val count = db.changes() + publishChange(emptyList(), changeLogModule.drain(db)) + count } - suspend fun deleteExpiredEvents() = pool.useWriter { expirationModule.deleteExpiredEvents(it) } + suspend fun deleteExpiredEvents() = + pool.useWriter { db -> + expirationModule.deleteExpiredEvents(db) + publishChange(emptyList(), changeLogModule.drain(db)) + } fun close() = pool.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StoreChange.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StoreChange.kt new file mode 100644 index 000000000..ef9742fb8 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StoreChange.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey + +/** + * One atomic batch of mutations that occurred inside the event store. + * + * The store emits exactly one [StoreChange] per successfully committed + * unit of work — `insertEvent`, a `transaction { ... }` block, + * `delete(...)`, `deleteExpiredEvents()`, or `clearDB()`. Empty changes + * (e.g. an insert that was rejected by a trigger) are not emitted. + * + * `removedIds` covers every row that left `event_headers` during the + * unit of work, regardless of cause: the supersession triggers for + * replaceable / addressable events, NIP-09 deletion fan-out, NIP-62 + * vanish cascades, NIP-40 expiration sweeps, and direct `delete(...)` + * calls. They are captured by an `AFTER DELETE` trigger that writes + * `OLD.id` into a per-connection TEMP table. + * + * `inserted` carries the events that survived the writer and are now + * in the database. A replaceable / addressable supersession therefore + * appears as a single change with `inserted = [new]` and + * `removedIds = [oldId]`. + */ +data class StoreChange( + val inserted: List, + val removedIds: List, +) { + fun isEmpty() = inserted.isEmpty() && removedIds.isEmpty() + + fun isNotEmpty() = !isEmpty() + + companion object { + val EMPTY = StoreChange(emptyList(), emptyList()) + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjectionTest.kt new file mode 100644 index 000000000..f9effa0ac --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjectionTest.kt @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent +import com.vitorpamplona.quartz.nip40Expiration.expiration +import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent +import com.vitorpamplona.quartz.utils.Secp256k1Instance +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotSame +import kotlin.test.assertSame +import kotlin.test.assertTrue + +class EventStoreProjectionTest { + private val signer = NostrSignerSync() + private lateinit var store: EventStore + private lateinit var scope: CoroutineScope + + @BeforeTest + fun setUp() { + Secp256k1Instance + store = EventStore(dbName = null) + scope = CoroutineScope(SupervisorJob()) + } + + @AfterTest + fun tearDown() { + scope.cancel() + store.close() + } + + /** + * Wait until [items][EventStoreProjection.items] reaches a state + * for which [predicate] is true. We poll the StateFlow rather than + * collect because the projection only re-emits on membership + * change — in-place addressable updates intentionally don't move + * the list reference. + */ + private suspend fun EventStoreProjection.awaitItems( + timeoutMs: Long = 5_000, + predicate: (List>) -> Boolean, + ): List> = + withTimeout(timeoutMs) { + items.first { predicate(it) } + } + + /** + * Wait until [flow] reaches [expected]. Used to observe an + * in-place addressable update where the list reference doesn't + * change but the slot's value does. + */ + private suspend fun awaitFlow( + flow: MutableStateFlow, + timeoutMs: Long = 5_000, + predicate: (T) -> Boolean, + ): T = + withTimeout(timeoutMs) { + flow.first { predicate(it) } + } + + @Test + fun seedReturnsExistingEvents() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) + store.insert(a) + store.insert(b) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + + val items = projection.items.value + assertEquals(2, items.size) + // Sorted by created_at DESC. + assertEquals(b.id, items[0].value.id) + assertEquals(a.id, items[1].value.id) + projection.close() + } + + @Test + fun insertAddsNewSlot() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + store.insert(a) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + val before = projection.items.value + assertEquals(1, before.size) + + val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) + store.insert(b) + + val after = projection.awaitItems { it.size == 2 } + assertNotSame(before, after, "insert must produce a new list reference") + assertEquals(b.id, after[0].value.id) + assertEquals(a.id, after[1].value.id) + projection.close() + } + + @Test + fun insertingNonMatchingEventDoesNotChangeList() = + runBlocking { + val text = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + store.insert(text) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + val seed = projection.items.value + + // Metadata kind doesn't match the filter. + val meta = signer.sign(MetadataEvent.createNew("Vitor", createdAt = 200)) + store.insert(meta) + + // Give the projection time to process the change. + delay(150) + assertSame(seed, projection.items.value) + projection.close() + } + + @Test + fun replaceableUpdateMutatesSlotInPlace() = + runBlocking { + val time = TimeUtils.now() + val v1 = signer.sign(MetadataEvent.createNew("v1", createdAt = time)) + store.insert(v1) + + val projection = + store.observe( + Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), + scope, + ) + projection.ready.await() + val seedList = projection.items.value + assertEquals(1, seedList.size) + val slot = seedList[0] + assertEquals(v1.id, slot.value.id) + + val v2 = signer.sign(MetadataEvent.createNew("v2", createdAt = time + 1)) + store.insert(v2) + + // The slot's flow updates... + awaitFlow(slot) { it.id == v2.id } + + // ...but the list reference is the SAME, and the slot is the SAME instance. + assertSame(seedList, projection.items.value, "addressable replace must not change list reference") + assertSame(slot, projection.items.value[0]) + projection.close() + } + + @Test + fun addressableUpdateMutatesSlotInPlace() = + runBlocking { + val time = TimeUtils.now() + val v1 = signer.sign(LongTextNoteEvent.build("blog v1", "title", dTag = "blog", createdAt = time)) + store.insert(v1) + + val projection = + store.observe( + Filter( + kinds = listOf(LongTextNoteEvent.KIND), + authors = listOf(v1.pubKey), + tags = mapOf("d" to listOf("blog")), + ), + scope, + ) + projection.ready.await() + val seedList = projection.items.value + assertEquals(1, seedList.size) + val slot = seedList[0] + assertEquals(v1.id, slot.value.id) + + val v2 = signer.sign(LongTextNoteEvent.build("blog v2", "title", dTag = "blog", createdAt = time + 1)) + store.insert(v2) + + awaitFlow(slot) { it.id == v2.id } + assertSame(seedList, projection.items.value, "addressable update must not change list reference") + assertSame(slot, projection.items.value[0]) + projection.close() + } + + @Test + fun nip09DeletionRemovesSlot() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) + store.insert(a) + store.insert(b) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + assertEquals(2, projection.items.value.size) + + val deletion = signer.sign(DeletionEvent.build(listOf(a))) + store.insert(deletion) + + val after = projection.awaitItems { it.size == 1 } + assertEquals(b.id, after[0].value.id) + projection.close() + } + + @Test + fun nip62VanishRemovesAllAuthorsEvents() = + runBlocking { + val time = TimeUtils.now() + val a = signer.sign(TextNoteEvent.build("a", createdAt = time)) + val b = signer.sign(TextNoteEvent.build("b", createdAt = time + 1)) + store.insert(a) + store.insert(b) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + assertEquals(2, projection.items.value.size) + + val vanish = + signer.sign( + RequestToVanishEvent.build( + "wss://quartz.local".normalizeRelayUrl(), + createdAt = time + 2, + ), + ) + store.insert(vanish) + + val after = projection.awaitItems { it.isEmpty() } + assertTrue(after.isEmpty()) + projection.close() + } + + @Test + fun nip40ExpirationRemovesSlotOnSweep() = + runBlocking { + val time = TimeUtils.now() + val safe = signer.sign(TextNoteEvent.build("safe", createdAt = time) { expiration(time + 100) }) + val short = signer.sign(TextNoteEvent.build("short", createdAt = time) { expiration(time + 1) }) + store.insert(safe) + store.insert(short) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + assertEquals(2, projection.items.value.size) + + // Wait for the expiration to lapse, then run the sweep. + delay(2000) + store.deleteExpiredEvents() + + val after = projection.awaitItems { it.size == 1 } + assertEquals(safe.id, after[0].value.id) + projection.close() + } + + @Test + fun manualDeleteByIdRemovesSlot() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + store.insert(a) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + assertEquals(1, projection.items.value.size) + + store.store.delete(a.id) + + val after = projection.awaitItems { it.isEmpty() } + assertTrue(after.isEmpty()) + projection.close() + } + + @Test + fun limitIsEnforcedOnInsertOverflow() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) + store.insert(a) + store.insert(b) + + val projection = + store.observe( + Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), + scope, + ) + projection.ready.await() + assertEquals(2, projection.items.value.size) + + // Newer event arrives; it should push the oldest out. + val c = signer.sign(TextNoteEvent.build("c", createdAt = 300)) + store.insert(c) + + val after = projection.awaitItems { it[0].value.id == c.id } + assertEquals(2, after.size) + assertEquals(c.id, after[0].value.id) + assertEquals(b.id, after[1].value.id) + projection.close() + } + + @Test + fun closeStopsListening() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + store.insert(a) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + projection.close() + + // Subsequent inserts must not surface in the (now empty) projection. + store.insert(signer.sign(TextNoteEvent.build("b", createdAt = 200))) + delay(150) + assertTrue(projection.items.value.isEmpty()) + } +} From c54f34ef08fa224d728f5ca6d8cb7ac2b45b76a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 19:56:00 +0000 Subject: [PATCH 02/24] refactor(quartz): make EventStoreProjection store-agnostic, NIP-aware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The projection now sits on top of any IEventStore, not just SQLite. Stores publish a simple `inserts: SharedFlow` (one event per successful insert), and the projection itself replays the NIP rules against that stream: - NIP-01 supersession with the lexical-id tiebreaker, for both replaceables (kind 0/3/10000-19999) and addressables (30000-39999). Both now share a single in-place update path. - NIP-09 deletions, with the original-author check (cross-author kind-5s are inert, matching the store). - NIP-62 right-to-vanish, scoped by the projection's relay arg. - NIP-40 expiration via a per-projection ticker that drops slots whose expiration tag has lapsed. Out-of-band store mutations (`delete(id)`, `clearDB()`, the periodic `deleteExpiredEvents()` sweep) are no longer visible to projections — they're maintenance ops; projections re-seed when their scope is restarted. Removed from SQLiteEventStore: - ChangeLogModule + temp-table + AFTER DELETE trigger. - StoreChange sealed type and the per-mutation drain/publish path. Both SQLiteEventStore and FsEventStore now satisfy `IEventStore.inserts`. FsEventStore.insertLocked returns Boolean so no-op idempotent retries (canonical already on disk) don't re-publish. Tests: - EventStoreProjectionTest moves to `store/projection/`, gains four new cases: NIP-01 out-of-order rejection, NIP-09 cross-author inertness, NIP-62 cross-author no-op, NIP-40 ticker-driven removal. - 13/13 projection tests + 232/232 total store tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../quartz/nip01Core/store/IEventStore.kt | 19 + .../store/projection/EventStoreProjection.kt | 401 ++++++++++++++++++ .../nip01Core/store/sqlite/ChangeLogModule.kt | 98 ----- .../nip01Core/store/sqlite/EventStore.kt | 16 +- .../store/sqlite/EventStoreProjection.kt | 247 ----------- .../store/sqlite/SQLiteEventStore.kt | 71 +--- .../nip01Core/store/sqlite/StoreChange.kt | 57 --- .../EventStoreProjectionTest.kt | 176 +++++--- .../quartz/nip01Core/store/fs/FsEventStore.kt | 59 ++- 9 files changed, 614 insertions(+), 530 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt delete mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ChangeLogModule.kt delete mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjection.kt delete mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StoreChange.kt rename quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/{sqlite => projection}/EventStoreProjectionTest.kt (70%) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index e70773c29..8a92f02c7 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.nip01Core.store import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import kotlinx.coroutines.flow.SharedFlow interface IEventStore : AutoCloseable { suspend fun insert(event: Event) @@ -56,5 +57,23 @@ interface IEventStore : AutoCloseable { suspend fun deleteExpiredEvents() + /** + * Stream of events the store accepted into durable storage. One + * emission per successfully inserted event, in commit order. + * Rejected inserts (expired, ephemeral, blocked by tombstone / + * vanish, NIP-01 supersession loser) emit nothing. + * + * Consumed by `EventStoreProjection` to maintain a live view — + * the projection itself replays NIP-01 supersession, NIP-09 + * deletion fan-out, NIP-62 vanish cascades, and NIP-40 expiration + * from these events, so stores don't need to publish removals. + * + * Out-of-band removals (`delete(id)`, `delete(filter)`, `clearDB()`, + * `deleteExpiredEvents()`) are not visible on this stream — they're + * maintenance operations and projections survive a missed mutation + * by re-seeding when their scope is restarted. + */ + val inserts: SharedFlow + override fun close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt new file mode 100644 index 000000000..3882b2db5 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt @@ -0,0 +1,401 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.projection + +import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.isAddressable +import com.vitorpamplona.quartz.nip01Core.core.isReplaceable +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent +import com.vitorpamplona.quartz.nip40Expiration.expiration +import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent +import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.yield + +/** + * A reactive projection over any [IEventStore] for a fixed set of + * [filters]. Each visible event is wrapped in a [MutableStateFlow] so + * the UI can collect three different kinds of change with the right + * granularity: + * + * - **Membership** (events arriving or leaving) re-emits a brand new + * [List] from [items]. The list reference is stable while membership + * is unchanged. + * - **In-place replaceable / addressable update** (a new version of + * the same `kind:pubkey:dtag` arrives) updates the existing handle's + * [MutableStateFlow.value] without touching the list. Only collectors + * of that one handle re-render. The list ordering is *not* + * reshuffled when the new version has a later `created_at` — each + * slot remembers the sort key it was inserted with, so updates feel + * like pure value mutations. + * - **Removal** (NIP-09 deletion, NIP-62 vanish, NIP-40 expiration) + * drops the handle from the list. + * + * The seed is materialised by querying the store once at start, after + * which the projection is driven entirely by [IEventStore.inserts] and + * its own expiration ticker. The store is never re-queried on + * mutation, and the projection never asks the store to delete anything + * — it interprets incoming Nostr events itself: + * + * - **NIP-01 supersession.** New replaceable / addressable events + * replace prior ones for the same `kind:pubkey[:dtag]`. The + * NIP-01 lexical-id tiebreaker (`new.id < old.id` when + * `created_at` ties) is honoured. + * - **NIP-09 deletions.** A [DeletionEvent] removes any matching + * handle owned by the same author (for GiftWrap, the recipient). + * Cross-author deletions are inert. + * - **NIP-62 right to vanish.** A [RequestToVanishEvent] whose + * `shouldVanishFrom([relay])` is true drops every handle from the + * same author with `created_at < vanish.created_at`. + * - **NIP-40 expiration.** Events with a past `expiration` tag are + * rejected at insert time. A periodic ticker drops slots whose + * expiration has just lapsed; collectors see the slot disappear. + * + * That's the same set of rules the SQLite / FS stores enforce for + * durability. The duplication is by design — the store enforces them + * on disk so the file isn't corrupt; the projection enforces them in + * memory so the live view stays correct without a re-query per event. + * + * Limit handling: the initial query honours the filter `limit`, and + * we trim to the same cap when an insert pushes the list over. We do + * **not** refill from the store after a deletion — if a deletion + * leaves you under the limit, you stay under the limit until something + * new arrives. That tradeoff matches what callers were already getting + * from `LocalCache.observeEvents`. + * + * Out-of-band store mutations — `delete(id)`, `delete(filter)`, + * `clearDB()`, the periodic `deleteExpiredEvents()` sweep — are not + * visible on [IEventStore.inserts] and won't update an open + * projection. Re-open the projection (e.g. cancel and recreate the + * scope) to pick up an out-of-band change. + * + * Lifecycle: the projection runs a collector + an expiration ticker + * inside [scope]. Cancel the scope (or call [close]) when the screen + * using the projection goes away. + */ +class EventStoreProjection( + private val store: IEventStore, + private val filters: List, + private val relay: NormalizedRelayUrl?, + scope: CoroutineScope, + private val expirationTickMs: Long = 30_000L, + private val nowProvider: () -> Long = TimeUtils::now, +) : AutoCloseable { + private val _items = MutableStateFlow>>(emptyList()) + val items: StateFlow>> = _items.asStateFlow() + + /** Slots keyed by the *current* event id. Re-keyed when a replaceable / addressable handle takes a new version. */ + private val byId = HashMap>() + + /** + * Slots keyed by `kind:pubkey:dtag` (or `kind:pubkey:` for plain + * replaceables) for in-place updates and supersession lookups. + */ + private val byStableKey = HashMap>() + + /** + * Sorted view of the same slots. The comparator uses each slot's + * frozen sort key (the seed event's `created_at` + `id`), so + * supersession in-place updates never move a slot inside this set. + */ + private val ordered = sortedSetOf(slotComparator()) + + private val limit: Int? = filters.mapNotNull { it.limit }.maxOrNull() + + /** Set when the seed has been written to [items], so callers can suspend until the projection is hot. */ + val ready: CompletableDeferred = CompletableDeferred() + + private val collectorJob: Job = + scope.launch { + seed() + ready.complete(Unit) + store.inserts.collect { event -> apply(event) } + } + + private val expirationJob: Job = + scope.launch { + // Sleep first so the seed-time sweep covers the initial + // contents — see [seed]. + while (true) { + delay(expirationTickMs) + sweepExpired() + } + } + + private suspend fun seed() { + val initial = store.query(filters) + val now = nowProvider() + for (event in initial) { + // The store should already exclude expired rows from the + // result, but it doesn't hurt to skip them here too — + // covers any FS / in-memory store that hasn't run a sweep + // recently. + if (isExpiredAt(event, now)) continue + insertNew(event) + yield() + } + publish() + } + + private fun apply(event: Event) { + if (isExpiredAt(event, nowProvider())) return + + var changed = false + + // Apply NIP-09 / NIP-62 side effects of the event before we + // consider matching the event itself against the filter — a + // deletion event that arrives at the same instant as a + // matching event still removes its targets. + if (event is DeletionEvent) { + if (handleDeletion(event)) changed = true + } + if (event is RequestToVanishEvent && event.shouldVanishFrom(relay)) { + if (handleVanish(event)) changed = true + } + + if (filters.any { it.match(event) }) { + if (handleInsert(event)) changed = true + } + + if (changed) publish() + } + + /** + * Returns true if the event matches the filter and its arrival + * caused membership to change (a fresh slot was added). Returns + * false when the arrival was an in-place supersession update or + * was rejected by the NIP-01 tiebreaker. + */ + @Suppress("UNCHECKED_CAST") + private fun handleInsert(event: Event): Boolean { + val key = stableKey(event) + if (key != null) { + val existing = byStableKey[key] + if (existing != null) { + if (!supersedes(event, existing.flow.value)) return false + + // Same address, new winner. Rekey byId and update the + // handle's value in place — list reference stays the + // same; only the handle's collectors re-render. + val previousId = existing.flow.value.id + if (previousId != event.id) { + byId.remove(previousId) + byId[event.id] = existing + } + existing.flow.value = event as T + return false + } + } else if (byId.containsKey(event.id)) { + return false + } + + insertNew(event) + return true + } + + private fun handleDeletion(deletion: DeletionEvent): Boolean { + var changed = false + + // NIP-09: delete by id, but only if the deletion's author owns + // the target. For GiftWrap, the owner is the p-tag recipient. + for (id in deletion.deleteEventIds()) { + val slot = byId[id] ?: continue + val ev = slot.flow.value + val owner = ownerPubKey(ev) + if (owner == deletion.pubKey && removeSlot(slot)) changed = true + } + + // NIP-09: delete by address, only original author, only events + // with `created_at <= deletion.created_at`. + for (addr in deletion.deleteAddresses()) { + if (addr.pubKeyHex != deletion.pubKey) continue + val key = stableKey(addr.kind, addr.pubKeyHex, addr.dTag) ?: continue + val slot = byStableKey[key] ?: continue + if (slot.flow.value.createdAt <= deletion.createdAt) { + if (removeSlot(slot)) changed = true + } + } + + return changed + } + + private fun handleVanish(vanish: RequestToVanishEvent): Boolean { + var changed = false + // Snapshot first because removeSlot mutates byId. + val targets = + byId.values.filter { + val ev = it.flow.value + ownerPubKey(ev) == vanish.pubKey && ev.createdAt < vanish.createdAt + } + for (slot in targets) { + if (removeSlot(slot)) changed = true + } + return changed + } + + private fun sweepExpired() { + val now = nowProvider() + val targets = byId.values.filter { isExpiredAt(it.flow.value, now) } + if (targets.isEmpty()) return + var changed = false + for (slot in targets) { + if (removeSlot(slot)) changed = true + } + if (changed) publish() + } + + @Suppress("UNCHECKED_CAST") + private fun insertNew(event: Event) { + val slot = Slot(event as T) + byId[event.id] = slot + stableKey(event)?.let { byStableKey[it] = slot } + ordered.add(slot) + + val cap = limit ?: return + while (ordered.size > cap) { + val tail = ordered.last() + removeSlot(tail) + } + } + + private fun removeSlot(slot: Slot): Boolean { + val removed = byId.remove(slot.flow.value.id) != null + if (!removed) return false + ordered.remove(slot) + stableKey(slot.flow.value)?.let { key -> + // Defensive: only clear the stable-key map if this slot + // still owns it. Could be stale after an addressable rekey + // raced with another insert. + if (byStableKey[key] === slot) byStableKey.remove(key) + } + return true + } + + private fun publish() { + _items.value = ordered.map { it.flow } + } + + /** + * Stop tracking changes and clear internal state. Idempotent. The + * scope passed to the constructor keeps running; only this + * projection's collector + expiration jobs are cancelled. + */ + override fun close() { + collectorJob.cancel() + expirationJob.cancel() + ordered.clear() + byId.clear() + byStableKey.clear() + _items.value = emptyList() + } + + /** + * Internal slot. Each event added to the projection lives inside + * one of these for as long as it survives. The sort key is frozen + * at construction time — supersession in-place updates rewrite + * `flow.value` but never the sort key, so the ordering inside + * [ordered] is stable across updates. + */ + private class Slot( + initial: T, + ) { + val sortCreatedAt: Long = initial.createdAt + val sortId: HexKey = initial.id + val flow: MutableStateFlow = MutableStateFlow(initial) + } + + companion object { + /** + * The lookup key for replaceable / addressable supersession. + * `null` for regular events (which only collide on event id). + */ + private fun stableKey(event: Event): String? = stableKey(event.kind, event.pubKey, (event as? AddressableEvent)?.dTag()) + + private fun stableKey( + kind: Int, + pubKeyHex: HexKey, + dTag: String?, + ): String? = + when { + kind.isAddressable() -> "$kind:$pubKeyHex:${dTag ?: ""}" + kind.isReplaceable() -> "$kind:$pubKeyHex:" + else -> null + } + + /** + * NIP-01 supersession tiebreaker. The new event wins iff: + * - its `created_at` is strictly greater, OR + * - the timestamps tie and its `id` is lexically smaller. + * Otherwise the existing slot keeps its place. + */ + private fun supersedes( + new: Event, + existing: Event, + ): Boolean = + when { + new.createdAt > existing.createdAt -> true + new.createdAt < existing.createdAt -> false + else -> new.id < existing.id + } + + /** + * Owner pubkey for ownership checks (NIP-09 author match, + * NIP-62 vanish target). For GiftWrap the owner is the p-tag + * recipient; for everything else it's `event.pubKey`. + */ + private fun ownerPubKey(event: Event): HexKey = (event as? GiftWrapEvent)?.recipientPubKey() ?: event.pubKey + + private fun isExpiredAt( + event: Event, + now: Long, + ): Boolean { + val exp = event.expiration() ?: return false + return exp <= now + } + + /** + * created_at DESC, id ASC. The keys are snapshots taken at + * insertion time, so the ordering of a slot never changes + * after it joins the set. Distinct events have distinct ids, + * so no third-key tiebreak is needed. + */ + private fun slotComparator(): Comparator> = + Comparator { a, b -> + if (a === b) return@Comparator 0 + val byTime = b.sortCreatedAt.compareTo(a.sortCreatedAt) + if (byTime != 0) return@Comparator byTime + a.sortId.compareTo(b.sortId) + } + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ChangeLogModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ChangeLogModule.kt deleted file mode 100644 index 73d34cd95..000000000 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ChangeLogModule.kt +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store.sqlite - -import androidx.sqlite.SQLiteConnection -import com.vitorpamplona.quartz.nip01Core.core.HexKey - -/** - * Per-connection scratch space that records the id of every row that - * leaves `event_headers`, regardless of why it left. - * - * Why TEMP: the table and trigger are connection-scoped, so they only - * exist on the writer connection where they're installed. Readers never - * see them and never accumulate rows. The data also lives only for the - * lifetime of the connection — exactly the right scope for "ids removed - * since the last drain." - * - * The trigger fires after every delete on `event_headers`, including: - * - the supersession triggers in [ReplaceableModule] / [AddressableModule], - * - the cascade from `event_vanish` in [RightToVanishModule], - * - the explicit deletes in [DeletionRequestModule], - * - [ExpirationModule.deleteExpiredEvents], - * - manual `delete(filter)` / `delete(id)`, - * - `clearDB()`. - * - * The [SQLiteEventStore] drains this log around each unit of work and - * publishes the resulting ids in a [StoreChange]. - */ -class ChangeLogModule { - fun installOnWriter(db: SQLiteConnection) { - db.execSQL( - """ - CREATE TEMP TABLE IF NOT EXISTS event_change_log ( - seq INTEGER PRIMARY KEY AUTOINCREMENT, - id TEXT NOT NULL - ) - """.trimIndent(), - ) - - db.execSQL( - """ - CREATE TEMP TRIGGER IF NOT EXISTS event_change_log_on_delete - AFTER DELETE ON event_headers - FOR EACH ROW - BEGIN - INSERT INTO event_change_log (id) VALUES (OLD.id); - END - """.trimIndent(), - ) - } - - /** - * Reads every id currently logged and clears the log. - * - * Must be called on the writer connection that owns the temp table. - * Callers are expected to be inside the writer mutex (drain happens - * inside or right after the `useWriter { ... }` block); a single - * read-and-clear pair is therefore atomic from the writer's view. - */ - fun drain(db: SQLiteConnection): List { - val ids = ArrayList() - db.prepare("SELECT id FROM event_change_log ORDER BY seq").use { stmt -> - while (stmt.step()) { - ids.add(stmt.getText(0)) - } - } - if (ids.isNotEmpty()) { - db.execSQL("DELETE FROM event_change_log") - } - return ids - } - - /** - * Drops everything from the log without returning it. Used after a - * rollback so the next successful unit of work starts clean. - */ - fun reset(db: SQLiteConnection) { - db.execSQL("DELETE FROM event_change_log") - } -} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index f17be61ee..092f10bcb 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -26,11 +26,12 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.projection.EventStoreProjection import kotlinx.coroutines.CoroutineScope class EventStore( dbName: String? = "events.db", - relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(), + val relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(), val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), ) : IEventStore { val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy) @@ -67,20 +68,17 @@ class EventStore( override suspend fun deleteExpiredEvents() = store.deleteExpiredEvents() - /** - * Stream of mutations committed to the store. See [SQLiteEventStore.changes]. - */ - val changes get() = store.changes + override val inserts get() = store.inserts /** - * Open a reactive [EventStoreProjection] over the store for - * [filters]. The projection runs inside [scope]; cancel the scope - * (or call [EventStoreProjection.close]) to release it. + * Open a reactive [EventStoreProjection] over this store with + * NIP-62 vanish scoping bound to the store's [relay]. Cancel + * [scope] (or call [EventStoreProjection.close]) to release it. */ fun observe( filters: List, scope: CoroutineScope, - ): EventStoreProjection = EventStoreProjection(store, filters, scope) + ): EventStoreProjection = EventStoreProjection(this, filters, relay, scope) fun observe( filter: Filter, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjection.kt deleted file mode 100644 index f655d6dfd..000000000 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjection.kt +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store.sqlite - -import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.HexKey -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.launch -import kotlinx.coroutines.yield - -/** - * A reactive projection over the [SQLiteEventStore] for a fixed set of - * [filters]. Each visible event is wrapped in a [MutableStateFlow] so - * the UI can collect three different kinds of change with the right - * granularity: - * - * - **Membership** (events arriving or leaving) re-emits a brand new - * [List] from [items]. The list reference is stable while membership - * is unchanged. - * - **In-place addressable update** (a new version of the same - * `kind:pubkey:dtag` arrives) updates the existing handle's - * [MutableStateFlow.value] without touching the list. Only collectors - * of that one handle re-render. The list ordering is *not* reshuffled - * when the new version has a later `created_at` — each slot remembers - * the sort key it was inserted with, so addressable updates feel like - * pure value mutations. - * - **Removal** (NIP-09 deletion, NIP-62 vanish, NIP-40 expiration, or - * a non-addressable event being explicitly deleted) drops the handle - * from the list. - * - * The seed is materialised by running the filters against the store - * once at start. After that, [items] is driven entirely by - * [SQLiteEventStore.changes]; the database is not re-queried on every - * mutation. - * - * Limit handling matches the existing in-memory observables in - * `commons/observables`: the initial query honours the filter `limit`, - * and we trim the list to the same cap when an insert pushes it over. - * We do **not** refill from the DB after a deletion — if a deletion - * leaves you under the limit, you stay under the limit until something - * new arrives. That tradeoff keeps the projection allocation-free per - * mutation and matches what callers were already getting from - * `LocalCache.observeEvents`. - * - * Lifecycle: the projection runs a single coroutine in [scope]. Cancel - * the scope (or call [close]) when the screen using the projection - * goes away. There is no shared state between projections; each one - * keeps its own indexes. - */ -class EventStoreProjection( - private val store: SQLiteEventStore, - private val filters: List, - scope: CoroutineScope, -) : AutoCloseable { - private val _items = MutableStateFlow>>(emptyList()) - val items: StateFlow>> = _items.asStateFlow() - - /** Slots keyed by the *current* event id. Re-keyed when an addressable handle takes a new version. */ - private val byId = HashMap>() - - /** Slots keyed by `kind:pubkey:dtag` for in-place addressable updates. */ - private val byAddress = HashMap>() - - /** - * Sorted view of the same slots. The comparator uses each slot's - * frozen sort key (the seed event's `created_at` + `id`), so an - * addressable update never moves a slot inside this set. - */ - private val ordered = sortedSetOf(slotComparator()) - - private val limit: Int? = filters.mapNotNull { it.limit }.maxOrNull() - - /** Set when the seed has been written to [items], so callers can suspend until the projection is hot. */ - val ready: CompletableDeferred = CompletableDeferred() - - private val job: Job = - scope.launch { - seed() - ready.complete(Unit) - store.changes.collect { change -> apply(change) } - } - - private suspend fun seed() { - val initial = store.query(filters) - for (event in initial) { - insertNew(event) - // Cooperate with cancellation on very large seeds. - yield() - } - publish() - } - - private fun apply(change: StoreChange) { - var changed = false - - // Removals first. A replaceable / addressable supersession - // arrives as `inserted = [new]` plus `removedIds = [oldId]`. - // When the new event is addressable and an existing slot - // already maps that address, [handleInsert] will rekey - // `byId` from the old id to the new id before this loop sees - // the old id — so the lookup here is a no-op for that case - // and the slot stays in place. For non-addressable - // replaceables (kinds 0/3/10000-19999) the old event has a - // different id, no address index, and we genuinely drop it. - for (event in change.inserted) { - if (handleInsert(event)) changed = true - } - for (id in change.removedIds) { - if (handleRemove(id)) changed = true - } - - if (changed) publish() - } - - @Suppress("UNCHECKED_CAST") - private fun handleInsert(event: Event): Boolean { - if (filters.none { it.match(event) }) return false - - if (event is AddressableEvent) { - val key = event.addressTag() - val existing = byAddress[key] - if (existing != null) { - // Same address, new version. Rekey byId from the - // previous event id to the new one and update the - // handle's value in place — list reference does not - // change, only the handle's collectors re-render. - val previousId = existing.flow.value.id - if (previousId != event.id) { - byId.remove(previousId) - byId[event.id] = existing - } - existing.flow.value = event as T - return false - } - } else if (byId.containsKey(event.id)) { - return false - } - - insertNew(event) - return true - } - - private fun handleRemove(id: HexKey): Boolean { - val slot = byId.remove(id) ?: return false - ordered.remove(slot) - val ev = slot.flow.value - if (ev is AddressableEvent) { - val addr = ev.addressTag() - // Only clear the address index if this slot still owns it. - if (byAddress[addr] === slot) byAddress.remove(addr) - } - return true - } - - @Suppress("UNCHECKED_CAST") - private fun insertNew(event: Event) { - val slot = Slot(event as T) - byId[event.id] = slot - if (event is AddressableEvent) byAddress[event.addressTag()] = slot - ordered.add(slot) - - val cap = limit ?: return - while (ordered.size > cap) { - val tail = ordered.last() - ordered.remove(tail) - val tailEvent = tail.flow.value - byId.remove(tailEvent.id) - if (tailEvent is AddressableEvent) { - val addr = tailEvent.addressTag() - if (byAddress[addr] === tail) byAddress.remove(addr) - } - } - } - - private fun publish() { - _items.value = ordered.map { it.flow } - } - - /** - * Stop tracking changes and clear internal state. Idempotent. The - * scope passed to the constructor keeps running; only this - * projection's collector job is cancelled. - */ - override fun close() { - job.cancel() - ordered.clear() - byId.clear() - byAddress.clear() - _items.value = emptyList() - } - - /** - * Internal slot. Each event added to the projection lives inside - * one of these for as long as it survives. The sort key is frozen - * at construction time — addressable in-place updates rewrite - * `flow.value` but never the sort key, so the ordering inside - * [ordered] is stable across updates. - */ - private class Slot( - initial: T, - ) { - val sortCreatedAt: Long = initial.createdAt - val sortId: HexKey = initial.id - val flow: MutableStateFlow = MutableStateFlow(initial) - } - - companion object { - /** - * created_at DESC, id ASC. The keys are snapshots taken at - * insertion time, so the ordering of a slot never changes - * after it joins the set. Distinct events have distinct ids, - * so no third-key tiebreak is needed. - */ - private fun slotComparator(): Comparator> = - Comparator { a, b -> - if (a === b) return@Comparator 0 - val byTime = b.sortCreatedAt.compareTo(a.sortCreatedAt) - if (byTime != 0) return@Comparator byTime - a.sortId.compareTo(b.sortId) - } - } -} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 83c95d531..9b3687baf 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -66,22 +66,18 @@ class SQLiteEventStore( val deletionModule = DeletionRequestModule(seedModule::hasher) val expirationModule = ExpirationModule() val rightToVanishModule = RightToVanishModule(seedModule::hasher) - val changeLogModule = ChangeLogModule() /** - * Stream of mutations committed to the store. One [StoreChange] per - * successful unit of work; rejected inserts and rolled-back - * transactions are not emitted. Subscribers see updates in commit - * order. See [EventStoreProjection] for a high-level reactive list - * that consumes this stream. + * Stream of events the store accepted into durable storage. See + * [IEventStore.inserts] for the contract. */ - private val _changes = - MutableSharedFlow( + private val _inserts = + MutableSharedFlow( replay = 0, extraBufferCapacity = 256, onBufferOverflow = BufferOverflow.SUSPEND, ) - val changes: SharedFlow = _changes.asSharedFlow() + val inserts: SharedFlow = _inserts.asSharedFlow() val queryBuilder = QueryBuilder( @@ -139,12 +135,6 @@ class SQLiteEventStore( setUserVersion(this, DATABASE_VERSION) } } - - // After the schema is in place (fresh or already-current - // DBs both reach this point), wire the change log onto - // the writer connection. TEMP table + TEMP trigger are - // connection-scoped, so this runs on the writer only. - changeLogModule.installOnWriter(db) }, ) } @@ -186,23 +176,10 @@ class SQLiteEventStore( } } - suspend fun clearDB() { + suspend fun clearDB() = pool.useWriter { db -> modules.reversed().forEach { it.deleteAll(db) } - // The deleteAll cascade fires the change-log trigger for - // every removed event_header row. Drain and publish so - // open projections drop everything they were holding. - publishChange(emptyList(), changeLogModule.drain(db)) } - } - - private suspend fun publishChange( - inserted: List, - removedIds: List, - ) { - if (inserted.isEmpty() && removedIds.isEmpty()) return - _changes.emit(StoreChange(inserted, removedIds)) - } suspend fun vacuum() = pool.useWriter { db -> @@ -237,12 +214,10 @@ class SQLiteEventStore( db.transaction { innerInsertEvent(event, this) } - // The transaction either committed or threw. On commit, the - // change log holds ids superseded by replaceable / addressable - // triggers and any NIP-09 / NIP-62 cascades fired by this - // event's content. On rollback the temp table was rolled back - // with the transaction, so drain returns empty. - publishChange(listOf(event), changeLogModule.drain(db)) + // The transaction either committed or threw. On commit the + // event is durable; emit so subscribed projections see it. + // On rollback the throw propagates and we never reach this. + _inserts.emit(event) } } @@ -268,7 +243,9 @@ class SQLiteEventStore( body() } } - publishChange(txn.accepted, changeLogModule.drain(db)) + // Emit each accepted event after the batch commits — the + // projection sees them in the same order they were inserted. + for (e in txn.accepted) _inserts.emit(e) } } @@ -308,31 +285,17 @@ class SQLiteEventStore( suspend fun count(filters: List): Int = pool.useReader { queryBuilder.count(filters, it) } - suspend fun delete(filter: Filter) = - pool.useWriter { db -> - queryBuilder.delete(filter, db) - publishChange(emptyList(), changeLogModule.drain(db)) - } + suspend fun delete(filter: Filter) = pool.useWriter { queryBuilder.delete(filter, it) } - suspend fun delete(filters: List) = - pool.useWriter { db -> - queryBuilder.delete(filters, db) - publishChange(emptyList(), changeLogModule.drain(db)) - } + suspend fun delete(filters: List) = pool.useWriter { queryBuilder.delete(filters, it) } suspend fun delete(id: HexKey): Int = pool.useWriter { db -> db.execSQL("DELETE FROM event_headers WHERE id = ?", arrayOf(id)) - val count = db.changes() - publishChange(emptyList(), changeLogModule.drain(db)) - count + db.changes() } - suspend fun deleteExpiredEvents() = - pool.useWriter { db -> - expirationModule.deleteExpiredEvents(db) - publishChange(emptyList(), changeLogModule.drain(db)) - } + suspend fun deleteExpiredEvents() = pool.useWriter { expirationModule.deleteExpiredEvents(it) } fun close() = pool.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StoreChange.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StoreChange.kt deleted file mode 100644 index ef9742fb8..000000000 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/StoreChange.kt +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store.sqlite - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.HexKey - -/** - * One atomic batch of mutations that occurred inside the event store. - * - * The store emits exactly one [StoreChange] per successfully committed - * unit of work — `insertEvent`, a `transaction { ... }` block, - * `delete(...)`, `deleteExpiredEvents()`, or `clearDB()`. Empty changes - * (e.g. an insert that was rejected by a trigger) are not emitted. - * - * `removedIds` covers every row that left `event_headers` during the - * unit of work, regardless of cause: the supersession triggers for - * replaceable / addressable events, NIP-09 deletion fan-out, NIP-62 - * vanish cascades, NIP-40 expiration sweeps, and direct `delete(...)` - * calls. They are captured by an `AFTER DELETE` trigger that writes - * `OLD.id` into a per-connection TEMP table. - * - * `inserted` carries the events that survived the writer and are now - * in the database. A replaceable / addressable supersession therefore - * appears as a single change with `inserted = [new]` and - * `removedIds = [oldId]`. - */ -data class StoreChange( - val inserted: List, - val removedIds: List, -) { - fun isEmpty() = inserted.isEmpty() && removedIds.isEmpty() - - fun isNotEmpty() = !isEmpty() - - companion object { - val EMPTY = StoreChange(emptyList(), emptyList()) - } -} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt similarity index 70% rename from quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjectionTest.kt rename to quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt index f9effa0ac..7644f1d31 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt @@ -18,13 +18,14 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.store.sqlite +package com.vitorpamplona.quartz.nip01Core.store.projection import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent @@ -50,6 +51,7 @@ import kotlin.test.assertTrue class EventStoreProjectionTest { private val signer = NostrSignerSync() + private val otherSigner = NostrSignerSync() private lateinit var store: EventStore private lateinit var scope: CoroutineScope @@ -66,13 +68,6 @@ class EventStoreProjectionTest { store.close() } - /** - * Wait until [items][EventStoreProjection.items] reaches a state - * for which [predicate] is true. We poll the StateFlow rather than - * collect because the projection only re-emits on membership - * change — in-place addressable updates intentionally don't move - * the list reference. - */ private suspend fun EventStoreProjection.awaitItems( timeoutMs: Long = 5_000, predicate: (List>) -> Boolean, @@ -81,11 +76,6 @@ class EventStoreProjectionTest { items.first { predicate(it) } } - /** - * Wait until [flow] reaches [expected]. Used to observe an - * in-place addressable update where the list reference doesn't - * change but the slot's value does. - */ private suspend fun awaitFlow( flow: MutableStateFlow, timeoutMs: Long = 5_000, @@ -108,7 +98,6 @@ class EventStoreProjectionTest { val items = projection.items.value assertEquals(2, items.size) - // Sorted by created_at DESC. assertEquals(b.id, items[0].value.id) assertEquals(a.id, items[1].value.id) projection.close() @@ -131,12 +120,11 @@ class EventStoreProjectionTest { val after = projection.awaitItems { it.size == 2 } assertNotSame(before, after, "insert must produce a new list reference") assertEquals(b.id, after[0].value.id) - assertEquals(a.id, after[1].value.id) projection.close() } @Test - fun insertingNonMatchingEventDoesNotChangeList() = + fun nonMatchingInsertDoesNotChangeList() = runBlocking { val text = signer.sign(TextNoteEvent.build("a", createdAt = 100)) store.insert(text) @@ -145,11 +133,9 @@ class EventStoreProjectionTest { projection.ready.await() val seed = projection.items.value - // Metadata kind doesn't match the filter. val meta = signer.sign(MetadataEvent.createNew("Vitor", createdAt = 200)) store.insert(meta) - // Give the projection time to process the change. delay(150) assertSame(seed, projection.items.value) projection.close() @@ -176,11 +162,8 @@ class EventStoreProjectionTest { val v2 = signer.sign(MetadataEvent.createNew("v2", createdAt = time + 1)) store.insert(v2) - // The slot's flow updates... awaitFlow(slot) { it.id == v2.id } - - // ...but the list reference is the SAME, and the slot is the SAME instance. - assertSame(seedList, projection.items.value, "addressable replace must not change list reference") + assertSame(seedList, projection.items.value, "replaceable update must not change list reference") assertSame(slot, projection.items.value[0]) projection.close() } @@ -203,16 +186,54 @@ class EventStoreProjectionTest { ) projection.ready.await() val seedList = projection.items.value - assertEquals(1, seedList.size) val slot = seedList[0] - assertEquals(v1.id, slot.value.id) val v2 = signer.sign(LongTextNoteEvent.build("blog v2", "title", dTag = "blog", createdAt = time + 1)) store.insert(v2) awaitFlow(slot) { it.id == v2.id } assertSame(seedList, projection.items.value, "addressable update must not change list reference") - assertSame(slot, projection.items.value[0]) + projection.close() + } + + /** + * Out-of-order arrival: the projection sees the *newer* version + * first (e.g. the relay sent v2 first), then v1. The projection + * must keep v2 in place and not regress to v1 — that's the NIP-01 + * supersession contract from the projection's side, since the + * store would have rejected v1 anyway. + */ + @Test + fun olderReplaceableArrivingAfterNewerIsRejected() = + runBlocking { + val time = TimeUtils.now() + val v1 = signer.sign(MetadataEvent.createNew("v1", createdAt = time)) + val v2 = signer.sign(MetadataEvent.createNew("v2", createdAt = time + 5)) + + // Seed the projection with v2 before v1 even hits the store + // — by inserting v2 first. + store.insert(v2) + + val projection = + store.observe( + Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), + scope, + ) + projection.ready.await() + val slot = projection.items.value[0] + assertEquals(v2.id, slot.value.id) + + // The store rejects v1 because v2 already won; the + // projection therefore never sees v1 on the inserts + // stream. The slot must still hold v2. + try { + store.insert(v1) + } catch (_: Throwable) { + // expected — store enforces the same rule + } + + delay(150) + assertEquals(v2.id, slot.value.id) projection.close() } @@ -224,7 +245,7 @@ class EventStoreProjectionTest { store.insert(a) store.insert(b) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() assertEquals(2, projection.items.value.size) @@ -236,8 +257,37 @@ class EventStoreProjectionTest { projection.close() } + /** + * NIP-09 cross-author deletions are inert. A different signer + * publishing a kind-5 targeting `a` must not drop the slot. + */ @Test - fun nip62VanishRemovesAllAuthorsEvents() = + fun nip09CrossAuthorDeletionIsInert() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + store.insert(a) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + val seed = projection.items.value + assertEquals(1, seed.size) + + val foreignDeletion = otherSigner.sign(DeletionEvent.build(listOf(a))) + store.insert(foreignDeletion) + + // Give the projection time to process the event. + delay(150) + assertSame(seed, projection.items.value) + assertEquals( + a.id, + projection.items.value[0] + .value.id, + ) + projection.close() + } + + @Test + fun nip62VanishRemovesAuthorEvents() = runBlocking { val time = TimeUtils.now() val a = signer.sign(TextNoteEvent.build("a", createdAt = time)) @@ -245,7 +295,7 @@ class EventStoreProjectionTest { store.insert(a) store.insert(b) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() assertEquals(2, projection.items.value.size) @@ -263,8 +313,42 @@ class EventStoreProjectionTest { projection.close() } + /** + * NIP-62 only removes events from the same author. A vanish from + * a different author must not touch slots owned by [signer]. + */ @Test - fun nip40ExpirationRemovesSlotOnSweep() = + fun nip62OtherAuthorVanishLeavesEventsAlone() = + runBlocking { + val time = TimeUtils.now() + val a = signer.sign(TextNoteEvent.build("a", createdAt = time)) + store.insert(a) + + val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + projection.ready.await() + val seed = projection.items.value + + val foreignVanish = + otherSigner.sign( + RequestToVanishEvent.build( + "wss://quartz.local".normalizeRelayUrl(), + createdAt = time + 2, + ), + ) + store.insert(foreignVanish) + + delay(150) + assertSame(seed, projection.items.value) + projection.close() + } + + /** + * NIP-40 per-projection ticker: a slot whose `expiration` lapses + * after the projection has loaded should be dropped on the next + * tick, even though the store hasn't run its sweep yet. + */ + @Test + fun nip40ExpirationDroppedByTicker() = runBlocking { val time = TimeUtils.now() val safe = signer.sign(TextNoteEvent.build("safe", createdAt = time) { expiration(time + 100) }) @@ -272,36 +356,26 @@ class EventStoreProjectionTest { store.insert(safe) store.insert(short) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + // Drive the ticker frequently so the test doesn't sit idle. + val projection = + EventStoreProjection( + store, + listOf(Filter(kinds = listOf(TextNoteEvent.KIND))), + relay = null, + scope = scope, + expirationTickMs = 100, + ) projection.ready.await() assertEquals(2, projection.items.value.size) - // Wait for the expiration to lapse, then run the sweep. + // Wait past the short expiration. delay(2000) - store.deleteExpiredEvents() - val after = projection.awaitItems { it.size == 1 } + val after = projection.awaitItems(timeoutMs = 5_000) { it.size == 1 } assertEquals(safe.id, after[0].value.id) projection.close() } - @Test - fun manualDeleteByIdRemovesSlot() = - runBlocking { - val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) - store.insert(a) - - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() - assertEquals(1, projection.items.value.size) - - store.store.delete(a.id) - - val after = projection.awaitItems { it.isEmpty() } - assertTrue(after.isEmpty()) - projection.close() - } - @Test fun limitIsEnforcedOnInsertOverflow() = runBlocking { @@ -318,7 +392,6 @@ class EventStoreProjectionTest { projection.ready.await() assertEquals(2, projection.items.value.size) - // Newer event arrives; it should push the oldest out. val c = signer.sign(TextNoteEvent.build("c", createdAt = 300)) store.insert(c) @@ -339,7 +412,6 @@ class EventStoreProjectionTest { projection.ready.await() projection.close() - // Subsequent inserts must not surface in the (now empty) projection. store.insert(signer.sign(TextNoteEvent.build("b", createdAt = 200))) delay(150) assertTrue(projection.items.value.isEmpty()) diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index 8da5d14b5..eec9265f1 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -36,6 +36,10 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip40Expiration.expiration import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.asSharedFlow import java.nio.file.FileAlreadyExistsException import java.nio.file.Files import java.nio.file.Path @@ -95,19 +99,40 @@ open class FsEventStore( planner = FsQueryPlanner(layout, hasher) } + /** + * Stream of events newly persisted by this store. See + * [IEventStore.inserts] for the contract. Events that are no-op + * idempotent retries (canonical already on disk from a previous + * call) are not re-emitted; the original insert already published. + */ + private val _inserts = + MutableSharedFlow( + replay = 0, + extraBufferCapacity = 256, + onBufferOverflow = BufferOverflow.SUSPEND, + ) + override val inserts: SharedFlow = _inserts.asSharedFlow() + // ------------------------------------------------------------------ // Insert // ------------------------------------------------------------------ - override suspend fun insert(event: Event) = - lockManager.withWriteLock { - insertLocked(event) - } + override suspend fun insert(event: Event) { + val accepted = lockManager.withWriteLock { insertLocked(event) } + if (accepted) _inserts.emit(event) + } - private fun insertLocked(event: Event) { - if (event.kind.isEphemeral()) return - if (isAlreadyExpired(event)) return - if (isBlockedByTombstone(event)) return + /** + * Inserts the event under the write lock and returns true iff this + * call is the one that newly persisted it. Returns false for the + * no-op paths (ephemeral, expired, blocked, supersession loser, or + * canonical already on disk from a prior call) so callers can + * decide whether to publish on [inserts]. + */ + private fun insertLocked(event: Event): Boolean { + if (event.kind.isEphemeral()) return false + if (isAlreadyExpired(event)) return false + if (isBlockedByTombstone(event)) return false val slot = slots.slotPathFor(event) val existingSlot = slot?.let { slots.readSlot(it) } @@ -121,7 +146,7 @@ open class FsEventStore( // (later createdAt, or same createdAt with the lexically smaller // id). Matches the ReplaceableModule / AddressableModule trigger // condition in SQLite. - return + return false } val canonical = layout.canonical(event.id) @@ -134,7 +159,7 @@ open class FsEventStore( } if (event is DeletionEvent) processDeletion(event, canonical) if (event is RequestToVanishEvent) processVanish(event, canonical) - return + return false } Files.createDirectories(canonical.parent) @@ -147,7 +172,7 @@ open class FsEventStore( // A concurrent writer won the race. Canonical is immutable // so the other copy is equivalent — drop our tmp and return. Files.deleteIfExists(tmp) - return + return false } Files.setLastModifiedTime(canonical, FileTime.from(event.createdAt, TimeUnit.SECONDS)) indexer.link(event, canonical) @@ -156,6 +181,7 @@ open class FsEventStore( } if (event is DeletionEvent) processDeletion(event, canonical) if (event is RequestToVanishEvent) processVanish(event, canonical) + return true } catch (t: Throwable) { Files.deleteIfExists(tmp) throw t @@ -263,14 +289,21 @@ open class FsEventStore( } } - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { + val accepted = ArrayList() lockManager.withWriteLock { val txn = object : IEventStore.ITransaction { - override fun insert(event: Event) = insertLocked(event) + override fun insert(event: Event) { + if (insertLocked(event)) accepted.add(event) + } } txn.body() } + // Emit each accepted event in order after the lock releases — + // mirrors SQLiteEventStore.transaction. + for (e in accepted) _inserts.emit(e) + } // ------------------------------------------------------------------ // Query From 86537edef585dbf97be80ab0fd94b020dcc24107 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 20:34:56 +0000 Subject: [PATCH 03/24] refactor(quartz): introduce ObservableEventStore so projections see ephemerals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits "events accepted for persistence" from "events accepted for observation". The new ObservableEventStore wraps any IEventStore and owns events: SharedFlow: - Non-ephemeral events forward to the inner store; success → emit, any rejection (expired, NIP-09 / NIP-62 tombstone, NIP-01 supersession loser) → no emit. - Ephemeral events (kinds 20000-29999) skip persistence entirely but still emit, so an open EventStoreProjection renders them while alive. They vanish from any future seed because the DB never had them. Reverts the previous round of plumbing inside SQLiteEventStore / FsEventStore: stores no longer carry an inserts SharedFlow. IEventStore goes back to a clean read/write contract. ObservableEventStore is the single place that decides what makes it onto the projection bus. EventStore (the SQLite convenience class) embeds an ObservableEventStore internally so existing call sites like `store.observe(filter, scope)` keep working without changes. EventStoreProjection now collects via Flow.onSubscription so the collector subscription is established before the seed query runs — fixes a race where an insert immediately after `ready.await()` could land in the SharedFlow before the collector was subscribed. Tests: - ephemeralEventsAppearInProjection — kind-22000 event reaches items but isn't queryable from the inner store, and a fresh projection on the same store gets an empty seed. - 14/14 projection tests + 219/219 other store tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../quartz/nip01Core/store/IEventStore.kt | 19 --- .../store/observable/ObservableEventStore.kt | 143 ++++++++++++++++++ .../store/projection/EventStoreProjection.kt | 40 +++-- .../nip01Core/store/sqlite/EventStore.kt | 66 +++++++- .../store/sqlite/SQLiteConnectionPool.kt | 4 +- .../store/sqlite/SQLiteEventStore.kt | 29 +--- .../projection/EventStoreProjectionTest.kt | 46 +++++- .../quartz/nip01Core/store/fs/FsEventStore.kt | 59 ++------ 8 files changed, 293 insertions(+), 113 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index 8a92f02c7..e70773c29 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -22,7 +22,6 @@ package com.vitorpamplona.quartz.nip01Core.store import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import kotlinx.coroutines.flow.SharedFlow interface IEventStore : AutoCloseable { suspend fun insert(event: Event) @@ -57,23 +56,5 @@ interface IEventStore : AutoCloseable { suspend fun deleteExpiredEvents() - /** - * Stream of events the store accepted into durable storage. One - * emission per successfully inserted event, in commit order. - * Rejected inserts (expired, ephemeral, blocked by tombstone / - * vanish, NIP-01 supersession loser) emit nothing. - * - * Consumed by `EventStoreProjection` to maintain a live view — - * the projection itself replays NIP-01 supersession, NIP-09 - * deletion fan-out, NIP-62 vanish cascades, and NIP-40 expiration - * from these events, so stores don't need to publish removals. - * - * Out-of-band removals (`delete(id)`, `delete(filter)`, `clearDB()`, - * `deleteExpiredEvents()`) are not visible on this stream — they're - * maintenance operations and projections survive a missed mutation - * by re-seeding when their scope is restarted. - */ - val inserts: SharedFlow - override fun close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt new file mode 100644 index 000000000..eeb53e22f --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.observable + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.isEphemeral +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip40Expiration.isExpired +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.asSharedFlow + +/** + * A reactive façade over any [IEventStore] that publishes every event + * accepted for *observation* — a superset of the events the inner + * store persists. + * + * The split between persistence and observation is the whole point of + * this class: + * + * - **Non-ephemeral events** are forwarded to the inner store. If the + * inner store rejects (expired, NIP-09 / NIP-62 tombstone, NIP-01 + * supersession loser), the rejection propagates and nothing is + * emitted on [events]. + * - **Ephemeral events** (kinds `20000-29999`) skip the inner store + * entirely — they're never persisted — but they still emit on + * [events] so projections can render them while they live. Already + * expired ephemerals are silently dropped. + * + * Wrap any store you want to observe — [SQLiteEventStore], FS-backed, + * an in-memory test fake — and feed [EventStoreProjection] from the + * resulting [events] flow. + * + * Reads (`query`, `count`) and out-of-band writes (`delete`, + * `deleteExpiredEvents`) forward to the inner store unchanged. The + * latter are *not* surfaced on [events] — see the projection's + * docstring for the rationale. + */ +class ObservableEventStore( + val inner: IEventStore, +) : IEventStore { + private val _events = + MutableSharedFlow( + replay = 0, + extraBufferCapacity = 256, + onBufferOverflow = BufferOverflow.SUSPEND, + ) + + /** + * Stream of events accepted for observation, persisted or not. + * One emission per successful [insert] (or per successful entry + * inside [transaction]); rejected inserts emit nothing. + */ + val events: SharedFlow = _events.asSharedFlow() + + override suspend fun insert(event: Event) { + if (event.kind.isEphemeral()) { + // Ephemeral kinds bypass persistence. We still drop ones + // that are already expired — they were never going to live + // long enough for a UI to render them. + if (event.isExpired()) return + _events.emit(event) + return + } + // Non-ephemeral: let the inner store enforce expiration, + // tombstones, supersession, etc. If it throws, we never emit. + inner.insert(event) + _events.emit(event) + } + + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { + val accepted = ArrayList() + inner.transaction { + val innerTxn = this + val wrapped = + object : IEventStore.ITransaction { + override fun insert(event: Event) { + if (event.kind.isEphemeral()) { + // Same ephemeral handling as the single-event + // path. The inner txn never sees these so the + // store's batch contains only persistable + // events. + if (event.isExpired()) return + accepted.add(event) + return + } + innerTxn.insert(event) + accepted.add(event) + } + } + wrapped.body() + } + // Emit only after the inner transaction commits. If it throws + // / rolls back, `accepted` is discarded. + for (e in accepted) _events.emit(e) + } + + override suspend fun query(filter: Filter): List = inner.query(filter) + + override suspend fun query(filters: List): List = inner.query(filters) + + override suspend fun query( + filter: Filter, + onEach: (T) -> Unit, + ) = inner.query(filter, onEach) + + override suspend fun query( + filters: List, + onEach: (T) -> Unit, + ) = inner.query(filters, onEach) + + override suspend fun count(filter: Filter): Int = inner.count(filter) + + override suspend fun count(filters: List): Int = inner.count(filters) + + override suspend fun delete(filter: Filter) = inner.delete(filter) + + override suspend fun delete(filters: List) = inner.delete(filters) + + override suspend fun deleteExpiredEvents() = inner.deleteExpiredEvents() + + override fun close() = inner.close() +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt index 3882b2db5..4e7c00bb4 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt @@ -27,7 +27,7 @@ import com.vitorpamplona.quartz.nip01Core.core.isAddressable import com.vitorpamplona.quartz.nip01Core.core.isReplaceable import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip40Expiration.expiration import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent @@ -40,12 +40,14 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.onSubscription import kotlinx.coroutines.launch import kotlinx.coroutines.yield /** - * A reactive projection over any [IEventStore] for a fixed set of - * [filters]. Each visible event is wrapped in a [MutableStateFlow] so + * A reactive projection over an [ObservableEventStore] for a fixed + * set of [filters]. Each visible event is wrapped in a + * [MutableStateFlow] so * the UI can collect three different kinds of change with the right * granularity: * @@ -63,10 +65,10 @@ import kotlinx.coroutines.yield * drops the handle from the list. * * The seed is materialised by querying the store once at start, after - * which the projection is driven entirely by [IEventStore.inserts] and - * its own expiration ticker. The store is never re-queried on - * mutation, and the projection never asks the store to delete anything - * — it interprets incoming Nostr events itself: + * which the projection is driven entirely by [ObservableEventStore.events] + * and its own expiration ticker. The store is never re-queried on + * mutation, and the projection never asks the store to delete + * anything — it interprets incoming Nostr events itself: * * - **NIP-01 supersession.** New replaceable / addressable events * replace prior ones for the same `kind:pubkey[:dtag]`. The @@ -96,16 +98,23 @@ import kotlinx.coroutines.yield * * Out-of-band store mutations — `delete(id)`, `delete(filter)`, * `clearDB()`, the periodic `deleteExpiredEvents()` sweep — are not - * visible on [IEventStore.inserts] and won't update an open + * visible on [ObservableEventStore.events] and won't update an open * projection. Re-open the projection (e.g. cancel and recreate the * scope) to pick up an out-of-band change. * + * Ephemeral events (kinds `20000-29999`) reach the projection via + * [ObservableEventStore.events] without ever being persisted; they + * appear in [items] for as long as the projection is alive but never + * survive a re-seed. NIP-40 expiration applies to them too — if they + * carry an `expiration` tag, the per-projection ticker drops them + * when it lapses. + * * Lifecycle: the projection runs a collector + an expiration ticker * inside [scope]. Cancel the scope (or call [close]) when the screen * using the projection goes away. */ class EventStoreProjection( - private val store: IEventStore, + private val store: ObservableEventStore, private val filters: List, private val relay: NormalizedRelayUrl?, scope: CoroutineScope, @@ -138,9 +147,16 @@ class EventStoreProjection( private val collectorJob: Job = scope.launch { - seed() - ready.complete(Unit) - store.inserts.collect { event -> apply(event) } + // `onSubscription` runs after the SharedFlow subscription is + // active but before we pull any events — the buffer absorbs + // any emissions that arrive while we seed, and we then drain + // them as the collect proceeds. Doing the seed inside + // `collect { }` instead would race with concurrent inserts. + store.events + .onSubscription { + seed() + ready.complete(Unit) + }.collect { event -> apply(event) } } private val expirationJob: Job = diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index 092f10bcb..9d188d9a2 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -26,19 +26,31 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore import com.vitorpamplona.quartz.nip01Core.store.projection.EventStoreProjection import kotlinx.coroutines.CoroutineScope +/** + * SQLite-backed event store with a built-in [ObservableEventStore] + * façade so [observe] returns an [EventStoreProjection] without + * extra plumbing. Persistence goes through [SQLiteEventStore]; the + * observable layer takes care of routing ephemerals (which never hit + * the DB) to projection collectors. + */ class EventStore( dbName: String? = "events.db", val relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(), val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), ) : IEventStore { val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy) + val observable = ObservableEventStore(SQLiteAdapter(store)) - override suspend fun insert(event: Event) = store.insertEvent(event) + /** Stream of events accepted for observation. See [ObservableEventStore.events]. */ + val events get() = observable.events - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = store.transaction(body) + override suspend fun insert(event: Event) = observable.insert(event) + + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = observable.transaction(body) override suspend fun query(filter: Filter) = store.query(filter) @@ -68,8 +80,6 @@ class EventStore( override suspend fun deleteExpiredEvents() = store.deleteExpiredEvents() - override val inserts get() = store.inserts - /** * Open a reactive [EventStoreProjection] over this store with * NIP-62 vanish scoping bound to the store's [relay]. Cancel @@ -78,7 +88,7 @@ class EventStore( fun observe( filters: List, scope: CoroutineScope, - ): EventStoreProjection = EventStoreProjection(this, filters, relay, scope) + ): EventStoreProjection = EventStoreProjection(observable, filters, relay, scope) fun observe( filter: Filter, @@ -87,3 +97,49 @@ class EventStore( override fun close() = store.close() } + +/** + * Adapts the non-IEventStore [SQLiteEventStore] to [IEventStore] for + * the [ObservableEventStore] wrapper. SQLiteEventStore predates the + * IEventStore contract; this is a thin forwarder, not a behavioural + * shim. + */ +private class SQLiteAdapter( + val sqlite: SQLiteEventStore, +) : IEventStore { + override suspend fun insert(event: Event) = sqlite.insertEvent(event) + + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { + sqlite.transaction { body() } + } + + override suspend fun query(filter: Filter) = sqlite.query(filter) + + override suspend fun query(filters: List) = sqlite.query(filters) + + override suspend fun query( + filter: Filter, + onEach: (T) -> Unit, + ) = sqlite.query(filter, onEach) + + override suspend fun query( + filters: List, + onEach: (T) -> Unit, + ) = sqlite.query(filters, onEach) + + override suspend fun count(filter: Filter) = sqlite.count(filter) + + override suspend fun count(filters: List) = sqlite.count(filters) + + override suspend fun delete(filter: Filter) { + sqlite.delete(filter) + } + + override suspend fun delete(filters: List) { + sqlite.delete(filters) + } + + override suspend fun deleteExpiredEvents() = sqlite.deleteExpiredEvents() + + override fun close() = sqlite.close() +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt index 2e4125b90..5ca19d985 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteConnectionPool.kt @@ -107,7 +107,7 @@ class SQLiteConnectionPool( * suspend until the lock is released. Cancellation-aware via the * coroutine [Mutex]. */ - suspend fun useWriter(block: suspend (SQLiteConnection) -> T): T = + suspend fun useWriter(block: (SQLiteConnection) -> T): T = writerMutex.withLock { block(writer) } @@ -118,7 +118,7 @@ class SQLiteConnectionPool( * (WAL). With an in-memory DB this falls back to the writer mutex * because each `:memory:` connection would be a separate database. */ - suspend fun useReader(block: suspend (SQLiteConnection) -> T): T { + suspend fun useReader(block: (SQLiteConnection) -> T): T { val ch = readerChannel ?: return writerMutex.withLock { block(writer) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 9b3687baf..2da5cf49c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -34,10 +34,6 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip40Expiration.isExpired import com.vitorpamplona.quartz.utils.EventFactory -import kotlinx.coroutines.channels.BufferOverflow -import kotlinx.coroutines.flow.MutableSharedFlow -import kotlinx.coroutines.flow.SharedFlow -import kotlinx.coroutines.flow.asSharedFlow class SQLiteEventStore( val driver: SQLiteDriver = BundledSQLiteDriver(), @@ -67,18 +63,6 @@ class SQLiteEventStore( val expirationModule = ExpirationModule() val rightToVanishModule = RightToVanishModule(seedModule::hasher) - /** - * Stream of events the store accepted into durable storage. See - * [IEventStore.inserts] for the contract. - */ - private val _inserts = - MutableSharedFlow( - replay = 0, - extraBufferCapacity = 256, - onBufferOverflow = BufferOverflow.SUSPEND, - ) - val inserts: SharedFlow = _inserts.asSharedFlow() - val queryBuilder = QueryBuilder( fullTextSearchModule, @@ -214,38 +198,27 @@ class SQLiteEventStore( db.transaction { innerInsertEvent(event, this) } - // The transaction either committed or threw. On commit the - // event is durable; emit so subscribed projections see it. - // On rollback the throw propagates and we never reach this. - _inserts.emit(event) } } inner class Transaction( val db: SQLiteConnection, ) : IEventStore.ITransaction { - val accepted = ArrayList() - override fun insert(event: Event) { if (event.isExpired()) throw SQLiteException("blocked: Cannot insert an expired event") if (event.kind.isEphemeral()) return innerInsertEvent(event, db) - accepted.add(event) } } suspend fun transaction(body: Transaction.() -> Unit) { pool.useWriter { db -> - val txn = Transaction(db) db.transaction { - with(txn) { + with(Transaction(this)) { body() } } - // Emit each accepted event after the batch commits — the - // projection sees them in the same order they were inserted. - for (e in txn.accepted) _inserts.emit(e) } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt index 7644f1d31..550c03b1b 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt @@ -359,7 +359,7 @@ class EventStoreProjectionTest { // Drive the ticker frequently so the test doesn't sit idle. val projection = EventStoreProjection( - store, + store.observable, listOf(Filter(kinds = listOf(TextNoteEvent.KIND))), relay = null, scope = scope, @@ -416,4 +416,48 @@ class EventStoreProjectionTest { delay(150) assertTrue(projection.items.value.isEmpty()) } + + /** + * Ephemeral events (kinds `20000-29999`) are never persisted, so + * the inner SQLite store silently drops them. The + * `ObservableEventStore` wrapper still routes them onto its + * [events][com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore.events] + * flow, so an open projection sees them while it's alive. They + * vanish from any future seed because the DB never had them. + */ + @Test + fun ephemeralEventsAppearInProjection() = + runBlocking { + val ephemeralKind = 22_000 + val projection = + store.observe(Filter(kinds = listOf(ephemeralKind)), scope) + projection.ready.await() + assertTrue(projection.items.value.isEmpty()) + + val ephemeral: Event = + signer.sign( + TimeUtils.now(), + ephemeralKind, + arrayOf(emptyArray()), + "live", + ) + store.insert(ephemeral) + + val after = projection.awaitItems { it.size == 1 } + assertEquals(ephemeral.id, after[0].value.id) + + // Confirmation that the inner SQLite store didn't persist. + val persisted = store.query(Filter(kinds = listOf(ephemeralKind))) + assertEquals(0, persisted.size) + + // A fresh projection on the same store gets nothing — the + // event was only ever live, not durable. + val freshProjection = + store.observe(Filter(kinds = listOf(ephemeralKind)), scope) + freshProjection.ready.await() + assertTrue(freshProjection.items.value.isEmpty()) + + projection.close() + freshProjection.close() + } } diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index eec9265f1..8da5d14b5 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -36,10 +36,6 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip40Expiration.expiration import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.TimeUtils -import kotlinx.coroutines.channels.BufferOverflow -import kotlinx.coroutines.flow.MutableSharedFlow -import kotlinx.coroutines.flow.SharedFlow -import kotlinx.coroutines.flow.asSharedFlow import java.nio.file.FileAlreadyExistsException import java.nio.file.Files import java.nio.file.Path @@ -99,40 +95,19 @@ open class FsEventStore( planner = FsQueryPlanner(layout, hasher) } - /** - * Stream of events newly persisted by this store. See - * [IEventStore.inserts] for the contract. Events that are no-op - * idempotent retries (canonical already on disk from a previous - * call) are not re-emitted; the original insert already published. - */ - private val _inserts = - MutableSharedFlow( - replay = 0, - extraBufferCapacity = 256, - onBufferOverflow = BufferOverflow.SUSPEND, - ) - override val inserts: SharedFlow = _inserts.asSharedFlow() - // ------------------------------------------------------------------ // Insert // ------------------------------------------------------------------ - override suspend fun insert(event: Event) { - val accepted = lockManager.withWriteLock { insertLocked(event) } - if (accepted) _inserts.emit(event) - } + override suspend fun insert(event: Event) = + lockManager.withWriteLock { + insertLocked(event) + } - /** - * Inserts the event under the write lock and returns true iff this - * call is the one that newly persisted it. Returns false for the - * no-op paths (ephemeral, expired, blocked, supersession loser, or - * canonical already on disk from a prior call) so callers can - * decide whether to publish on [inserts]. - */ - private fun insertLocked(event: Event): Boolean { - if (event.kind.isEphemeral()) return false - if (isAlreadyExpired(event)) return false - if (isBlockedByTombstone(event)) return false + private fun insertLocked(event: Event) { + if (event.kind.isEphemeral()) return + if (isAlreadyExpired(event)) return + if (isBlockedByTombstone(event)) return val slot = slots.slotPathFor(event) val existingSlot = slot?.let { slots.readSlot(it) } @@ -146,7 +121,7 @@ open class FsEventStore( // (later createdAt, or same createdAt with the lexically smaller // id). Matches the ReplaceableModule / AddressableModule trigger // condition in SQLite. - return false + return } val canonical = layout.canonical(event.id) @@ -159,7 +134,7 @@ open class FsEventStore( } if (event is DeletionEvent) processDeletion(event, canonical) if (event is RequestToVanishEvent) processVanish(event, canonical) - return false + return } Files.createDirectories(canonical.parent) @@ -172,7 +147,7 @@ open class FsEventStore( // A concurrent writer won the race. Canonical is immutable // so the other copy is equivalent — drop our tmp and return. Files.deleteIfExists(tmp) - return false + return } Files.setLastModifiedTime(canonical, FileTime.from(event.createdAt, TimeUnit.SECONDS)) indexer.link(event, canonical) @@ -181,7 +156,6 @@ open class FsEventStore( } if (event is DeletionEvent) processDeletion(event, canonical) if (event is RequestToVanishEvent) processVanish(event, canonical) - return true } catch (t: Throwable) { Files.deleteIfExists(tmp) throw t @@ -289,21 +263,14 @@ open class FsEventStore( } } - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { - val accepted = ArrayList() + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = lockManager.withWriteLock { val txn = object : IEventStore.ITransaction { - override fun insert(event: Event) { - if (insertLocked(event)) accepted.add(event) - } + override fun insert(event: Event) = insertLocked(event) } txn.body() } - // Emit each accepted event in order after the lock releases — - // mirrors SQLiteEventStore.transaction. - for (e in accepted) _inserts.emit(e) - } // ------------------------------------------------------------------ // Query From 32850c3d4032d09c248690227b5811db1b772585 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 21:06:27 +0000 Subject: [PATCH 04/24] refactor(quartz): make ObservableEventStore an external composition layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the previous round's choice of embedding an ObservableEventStore inside EventStore. The wrapper now lives strictly outside any specific store: callers compose `ObservableEventStore(EventStore(...))` (or `ObservableEventStore(FsEventStore(...))`, or any IEventStore) and use that as their projection bus. EventStore is back to a plain IEventStore implementation over SQLiteEventStore — no `observable`, `events`, or `observe()` on it. ObservableEventStore gains the `observe(filters, relay, scope)` and `observe(filter, relay, scope)` convenience methods. NIP-62 vanish scoping is passed in per-projection (rather than a constructor field on the wrapper) so the same observable can feed projections with different relay scopes. Tests construct `observable = ObservableEventStore(store)` in `setUp` and route inserts through `observable.insert(...)` so the projection bus actually publishes them. 14/14 projection + 219/219 other store tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../store/observable/ObservableEventStore.kt | 22 +++++ .../nip01Core/store/sqlite/EventStore.kt | 80 +-------------- .../projection/EventStoreProjectionTest.kt | 97 ++++++++++--------- 3 files changed, 79 insertions(+), 120 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt index eeb53e22f..7757b9fba 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt @@ -23,8 +23,11 @@ package com.vitorpamplona.quartz.nip01Core.store.observable import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.isEphemeral import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.projection.EventStoreProjection import com.vitorpamplona.quartz.nip40Expiration.isExpired +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.SharedFlow @@ -139,5 +142,24 @@ class ObservableEventStore( override suspend fun deleteExpiredEvents() = inner.deleteExpiredEvents() + /** + * Open a reactive [EventStoreProjection] over this observable + * store. [relay] scopes NIP-62 vanish handling — pass the relay + * URL the events are arriving from, or `null` to apply only + * unscoped (`ALL_RELAYS`) vanish requests. Cancel [scope] (or + * call [EventStoreProjection.close]) to release the projection. + */ + fun observe( + filters: List, + relay: NormalizedRelayUrl?, + scope: CoroutineScope, + ): EventStoreProjection = EventStoreProjection(this, filters, relay, scope) + + fun observe( + filter: Filter, + relay: NormalizedRelayUrl?, + scope: CoroutineScope, + ): EventStoreProjection = observe(listOf(filter), relay, scope) + override fun close() = inner.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index 9d188d9a2..0a0f1a75b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -26,16 +26,11 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore -import com.vitorpamplona.quartz.nip01Core.store.projection.EventStoreProjection -import kotlinx.coroutines.CoroutineScope /** - * SQLite-backed event store with a built-in [ObservableEventStore] - * façade so [observe] returns an [EventStoreProjection] without - * extra plumbing. Persistence goes through [SQLiteEventStore]; the - * observable layer takes care of routing ephemerals (which never hit - * the DB) to projection collectors. + * SQLite-backed [IEventStore] with default DB-file name and relay + * scoping. Wrap in `ObservableEventStore` if you want to feed + * `EventStoreProjection`. */ class EventStore( dbName: String? = "events.db", @@ -43,14 +38,10 @@ class EventStore( val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), ) : IEventStore { val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy) - val observable = ObservableEventStore(SQLiteAdapter(store)) - /** Stream of events accepted for observation. See [ObservableEventStore.events]. */ - val events get() = observable.events + override suspend fun insert(event: Event) = store.insertEvent(event) - override suspend fun insert(event: Event) = observable.insert(event) - - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = observable.transaction(body) + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = store.transaction(body) override suspend fun query(filter: Filter) = store.query(filter) @@ -80,66 +71,5 @@ class EventStore( override suspend fun deleteExpiredEvents() = store.deleteExpiredEvents() - /** - * Open a reactive [EventStoreProjection] over this store with - * NIP-62 vanish scoping bound to the store's [relay]. Cancel - * [scope] (or call [EventStoreProjection.close]) to release it. - */ - fun observe( - filters: List, - scope: CoroutineScope, - ): EventStoreProjection = EventStoreProjection(observable, filters, relay, scope) - - fun observe( - filter: Filter, - scope: CoroutineScope, - ): EventStoreProjection = observe(listOf(filter), scope) - override fun close() = store.close() } - -/** - * Adapts the non-IEventStore [SQLiteEventStore] to [IEventStore] for - * the [ObservableEventStore] wrapper. SQLiteEventStore predates the - * IEventStore contract; this is a thin forwarder, not a behavioural - * shim. - */ -private class SQLiteAdapter( - val sqlite: SQLiteEventStore, -) : IEventStore { - override suspend fun insert(event: Event) = sqlite.insertEvent(event) - - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { - sqlite.transaction { body() } - } - - override suspend fun query(filter: Filter) = sqlite.query(filter) - - override suspend fun query(filters: List) = sqlite.query(filters) - - override suspend fun query( - filter: Filter, - onEach: (T) -> Unit, - ) = sqlite.query(filter, onEach) - - override suspend fun query( - filters: List, - onEach: (T) -> Unit, - ) = sqlite.query(filters, onEach) - - override suspend fun count(filter: Filter) = sqlite.count(filter) - - override suspend fun count(filters: List) = sqlite.count(filters) - - override suspend fun delete(filter: Filter) { - sqlite.delete(filter) - } - - override suspend fun delete(filters: List) { - sqlite.delete(filters) - } - - override suspend fun deleteExpiredEvents() = sqlite.deleteExpiredEvents() - - override fun close() = sqlite.close() -} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt index 550c03b1b..182847661 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent @@ -53,12 +54,14 @@ class EventStoreProjectionTest { private val signer = NostrSignerSync() private val otherSigner = NostrSignerSync() private lateinit var store: EventStore + private lateinit var observable: ObservableEventStore private lateinit var scope: CoroutineScope @BeforeTest fun setUp() { Secp256k1Instance store = EventStore(dbName = null) + observable = ObservableEventStore(store) scope = CoroutineScope(SupervisorJob()) } @@ -90,10 +93,10 @@ class EventStoreProjectionTest { runBlocking { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) - store.insert(a) - store.insert(b) + observable.insert(a) + observable.insert(b) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) projection.ready.await() val items = projection.items.value @@ -107,15 +110,15 @@ class EventStoreProjectionTest { fun insertAddsNewSlot() = runBlocking { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) - store.insert(a) + observable.insert(a) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) projection.ready.await() val before = projection.items.value assertEquals(1, before.size) val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) - store.insert(b) + observable.insert(b) val after = projection.awaitItems { it.size == 2 } assertNotSame(before, after, "insert must produce a new list reference") @@ -127,14 +130,14 @@ class EventStoreProjectionTest { fun nonMatchingInsertDoesNotChangeList() = runBlocking { val text = signer.sign(TextNoteEvent.build("a", createdAt = 100)) - store.insert(text) + observable.insert(text) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) projection.ready.await() val seed = projection.items.value val meta = signer.sign(MetadataEvent.createNew("Vitor", createdAt = 200)) - store.insert(meta) + observable.insert(meta) delay(150) assertSame(seed, projection.items.value) @@ -146,11 +149,12 @@ class EventStoreProjectionTest { runBlocking { val time = TimeUtils.now() val v1 = signer.sign(MetadataEvent.createNew("v1", createdAt = time)) - store.insert(v1) + observable.insert(v1) val projection = - store.observe( + observable.observe( Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), + store.relay, scope, ) projection.ready.await() @@ -160,7 +164,7 @@ class EventStoreProjectionTest { assertEquals(v1.id, slot.value.id) val v2 = signer.sign(MetadataEvent.createNew("v2", createdAt = time + 1)) - store.insert(v2) + observable.insert(v2) awaitFlow(slot) { it.id == v2.id } assertSame(seedList, projection.items.value, "replaceable update must not change list reference") @@ -173,15 +177,16 @@ class EventStoreProjectionTest { runBlocking { val time = TimeUtils.now() val v1 = signer.sign(LongTextNoteEvent.build("blog v1", "title", dTag = "blog", createdAt = time)) - store.insert(v1) + observable.insert(v1) val projection = - store.observe( + observable.observe( Filter( kinds = listOf(LongTextNoteEvent.KIND), authors = listOf(v1.pubKey), tags = mapOf("d" to listOf("blog")), ), + store.relay, scope, ) projection.ready.await() @@ -189,7 +194,7 @@ class EventStoreProjectionTest { val slot = seedList[0] val v2 = signer.sign(LongTextNoteEvent.build("blog v2", "title", dTag = "blog", createdAt = time + 1)) - store.insert(v2) + observable.insert(v2) awaitFlow(slot) { it.id == v2.id } assertSame(seedList, projection.items.value, "addressable update must not change list reference") @@ -212,11 +217,12 @@ class EventStoreProjectionTest { // Seed the projection with v2 before v1 even hits the store // — by inserting v2 first. - store.insert(v2) + observable.insert(v2) val projection = - store.observe( + observable.observe( Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), + store.relay, scope, ) projection.ready.await() @@ -227,7 +233,7 @@ class EventStoreProjectionTest { // projection therefore never sees v1 on the inserts // stream. The slot must still hold v2. try { - store.insert(v1) + observable.insert(v1) } catch (_: Throwable) { // expected — store enforces the same rule } @@ -242,15 +248,15 @@ class EventStoreProjectionTest { runBlocking { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) - store.insert(a) - store.insert(b) + observable.insert(a) + observable.insert(b) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) projection.ready.await() assertEquals(2, projection.items.value.size) val deletion = signer.sign(DeletionEvent.build(listOf(a))) - store.insert(deletion) + observable.insert(deletion) val after = projection.awaitItems { it.size == 1 } assertEquals(b.id, after[0].value.id) @@ -265,15 +271,15 @@ class EventStoreProjectionTest { fun nip09CrossAuthorDeletionIsInert() = runBlocking { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) - store.insert(a) + observable.insert(a) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) projection.ready.await() val seed = projection.items.value assertEquals(1, seed.size) val foreignDeletion = otherSigner.sign(DeletionEvent.build(listOf(a))) - store.insert(foreignDeletion) + observable.insert(foreignDeletion) // Give the projection time to process the event. delay(150) @@ -292,10 +298,10 @@ class EventStoreProjectionTest { val time = TimeUtils.now() val a = signer.sign(TextNoteEvent.build("a", createdAt = time)) val b = signer.sign(TextNoteEvent.build("b", createdAt = time + 1)) - store.insert(a) - store.insert(b) + observable.insert(a) + observable.insert(b) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) projection.ready.await() assertEquals(2, projection.items.value.size) @@ -306,7 +312,7 @@ class EventStoreProjectionTest { createdAt = time + 2, ), ) - store.insert(vanish) + observable.insert(vanish) val after = projection.awaitItems { it.isEmpty() } assertTrue(after.isEmpty()) @@ -322,9 +328,9 @@ class EventStoreProjectionTest { runBlocking { val time = TimeUtils.now() val a = signer.sign(TextNoteEvent.build("a", createdAt = time)) - store.insert(a) + observable.insert(a) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) projection.ready.await() val seed = projection.items.value @@ -335,7 +341,7 @@ class EventStoreProjectionTest { createdAt = time + 2, ), ) - store.insert(foreignVanish) + observable.insert(foreignVanish) delay(150) assertSame(seed, projection.items.value) @@ -353,13 +359,13 @@ class EventStoreProjectionTest { val time = TimeUtils.now() val safe = signer.sign(TextNoteEvent.build("safe", createdAt = time) { expiration(time + 100) }) val short = signer.sign(TextNoteEvent.build("short", createdAt = time) { expiration(time + 1) }) - store.insert(safe) - store.insert(short) + observable.insert(safe) + observable.insert(short) // Drive the ticker frequently so the test doesn't sit idle. val projection = EventStoreProjection( - store.observable, + observable, listOf(Filter(kinds = listOf(TextNoteEvent.KIND))), relay = null, scope = scope, @@ -381,19 +387,20 @@ class EventStoreProjectionTest { runBlocking { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) - store.insert(a) - store.insert(b) + observable.insert(a) + observable.insert(b) val projection = - store.observe( + observable.observe( Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), + store.relay, scope, ) projection.ready.await() assertEquals(2, projection.items.value.size) val c = signer.sign(TextNoteEvent.build("c", createdAt = 300)) - store.insert(c) + observable.insert(c) val after = projection.awaitItems { it[0].value.id == c.id } assertEquals(2, after.size) @@ -406,13 +413,13 @@ class EventStoreProjectionTest { fun closeStopsListening() = runBlocking { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) - store.insert(a) + observable.insert(a) - val projection = store.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) projection.ready.await() projection.close() - store.insert(signer.sign(TextNoteEvent.build("b", createdAt = 200))) + observable.insert(signer.sign(TextNoteEvent.build("b", createdAt = 200))) delay(150) assertTrue(projection.items.value.isEmpty()) } @@ -430,7 +437,7 @@ class EventStoreProjectionTest { runBlocking { val ephemeralKind = 22_000 val projection = - store.observe(Filter(kinds = listOf(ephemeralKind)), scope) + observable.observe(Filter(kinds = listOf(ephemeralKind)), store.relay, scope) projection.ready.await() assertTrue(projection.items.value.isEmpty()) @@ -441,7 +448,7 @@ class EventStoreProjectionTest { arrayOf(emptyArray()), "live", ) - store.insert(ephemeral) + observable.insert(ephemeral) val after = projection.awaitItems { it.size == 1 } assertEquals(ephemeral.id, after[0].value.id) @@ -453,7 +460,7 @@ class EventStoreProjectionTest { // A fresh projection on the same store gets nothing — the // event was only ever live, not durable. val freshProjection = - store.observe(Filter(kinds = listOf(ephemeralKind)), scope) + observable.observe(Filter(kinds = listOf(ephemeralKind)), store.relay, scope) freshProjection.ready.await() assertTrue(freshProjection.items.value.isEmpty()) From d394a38f2b7d928b7dfb58e75f886b1cd0845bc0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 21:20:31 +0000 Subject: [PATCH 05/24] feat(quartz): emit StoreEvent.Delete from observable for filter / expired sweeps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces ObservableEventStore.events: SharedFlow with SharedFlow, where StoreEvent is: Insert(event) Delete(rule: DeleteRule) DeleteRule.Filtered(filters) ← from delete(filter) / delete(filters) DeleteRule.Expired(asOf) ← from deleteExpiredEvents() ObservableEventStore now emits Insert for accepted events, and Delete for every out-of-band removal that went through it. The Expired rule carries the cutoff timestamp (TimeUtils.now() at call time) so the projection's in-memory drop matches the store's on-disk drop without clock skew between collectors. EventStoreProjection no longer runs its own NIP-40 ticker. Expired events linger in [items] until the application calls deleteExpiredEvents() on the observable — at which point the projection drops everything whose expiration is past `asOf`. The ticker, expirationTickMs constructor arg, and sweepExpired helper are all removed. For DeleteRule.Filtered the projection iterates current slots and drops any whose event matches any of the rule's filters via Filter.match — same predicate the store uses. Tests: - nip40ExpirationDroppedByTicker → nip40ExpirationDroppedOnStoreSweep: drives a deleteExpiredEvents() call instead of waiting on a ticker. - New deleteByFilterRemovesMatchingSlots: confirms delete(filter) on the observable removes exactly the matching slots from the projection without touching others. - 15/15 projection tests + all other store tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../store/observable/ObservableEventStore.kt | 43 ++++-- .../nip01Core/store/observable/StoreEvent.kt | 71 +++++++++ .../store/projection/EventStoreProjection.kt | 136 +++++++++--------- .../projection/EventStoreProjectionTest.kt | 59 ++++++-- 4 files changed, 220 insertions(+), 89 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/StoreEvent.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt index 7757b9fba..3f66839d2 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt @@ -27,6 +27,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.projection.EventStoreProjection import com.vitorpamplona.quartz.nip40Expiration.isExpired +import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.MutableSharedFlow @@ -63,18 +64,23 @@ class ObservableEventStore( val inner: IEventStore, ) : IEventStore { private val _events = - MutableSharedFlow( + MutableSharedFlow( replay = 0, extraBufferCapacity = 256, onBufferOverflow = BufferOverflow.SUSPEND, ) /** - * Stream of events accepted for observation, persisted or not. - * One emission per successful [insert] (or per successful entry - * inside [transaction]); rejected inserts emit nothing. + * Stream of mutations accepted by the observable layer. One + * emission per successful [insert] (or per accepted event in a + * [transaction] body), one emission per [delete] / [delete] / + * [deleteExpiredEvents] call. Rejected inserts and rolled-back + * transactions emit nothing. + * + * Projections consume this stream — see [EventStoreProjection] + * for how each [StoreEvent] is interpreted. */ - val events: SharedFlow = _events.asSharedFlow() + val events: SharedFlow = _events.asSharedFlow() override suspend fun insert(event: Event) { if (event.kind.isEphemeral()) { @@ -82,13 +88,13 @@ class ObservableEventStore( // that are already expired — they were never going to live // long enough for a UI to render them. if (event.isExpired()) return - _events.emit(event) + _events.emit(StoreEvent.Insert(event)) return } // Non-ephemeral: let the inner store enforce expiration, // tombstones, supersession, etc. If it throws, we never emit. inner.insert(event) - _events.emit(event) + _events.emit(StoreEvent.Insert(event)) } override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { @@ -115,7 +121,7 @@ class ObservableEventStore( } // Emit only after the inner transaction commits. If it throws // / rolls back, `accepted` is discarded. - for (e in accepted) _events.emit(e) + for (e in accepted) _events.emit(StoreEvent.Insert(e)) } override suspend fun query(filter: Filter): List = inner.query(filter) @@ -136,11 +142,26 @@ class ObservableEventStore( override suspend fun count(filters: List): Int = inner.count(filters) - override suspend fun delete(filter: Filter) = inner.delete(filter) + override suspend fun delete(filter: Filter) { + inner.delete(filter) + _events.emit(StoreEvent.Delete(DeleteRule.Filtered(listOf(filter)))) + } - override suspend fun delete(filters: List) = inner.delete(filters) + override suspend fun delete(filters: List) { + inner.delete(filters) + _events.emit(StoreEvent.Delete(DeleteRule.Filtered(filters))) + } - override suspend fun deleteExpiredEvents() = inner.deleteExpiredEvents() + override suspend fun deleteExpiredEvents() { + // Pin the cutoff before forwarding so the projection's drop + // matches the store's drop exactly. The store's SQL uses + // `unixepoch()` so there's still a small skew, but pinning at + // call time is closer than letting each projection use its + // own clock when the event is processed. + val asOf = TimeUtils.now() + inner.deleteExpiredEvents() + _events.emit(StoreEvent.Delete(DeleteRule.Expired(asOf))) + } /** * Open a reactive [EventStoreProjection] over this observable diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/StoreEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/StoreEvent.kt new file mode 100644 index 000000000..83d46d46a --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/StoreEvent.kt @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.observable + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter + +/** + * Mutations published by [ObservableEventStore.events]. Projections + * react to these to keep their in-memory view in sync with the + * underlying store. + * + * - [Insert] is emitted for every event accepted by the observable + * layer (persistable or ephemeral). Carries the event itself so + * the projection can run its NIP-01 / NIP-09 / NIP-62 + * interpretation. + * - [Delete] is emitted for every out-of-band removal — manual + * `delete(filter)` / `delete(filters)` calls and the periodic + * `deleteExpiredEvents()` sweep. Carries the rule the store used + * so the projection can apply the same selection in memory + * without re-querying. + */ +sealed interface StoreEvent { + data class Insert( + val event: Event, + ) : StoreEvent + + data class Delete( + val rule: DeleteRule, + ) : StoreEvent +} + +/** + * Selection rule for a [StoreEvent.Delete]. [Filtered] mirrors the + * `delete(filter)` / `delete(filters)` API and OR-combines the + * filters; [Expired] mirrors `deleteExpiredEvents()` and removes + * everything whose NIP-40 expiration has lapsed. + */ +sealed interface DeleteRule { + data class Filtered( + val filters: List, + ) : DeleteRule + + /** + * NIP-40 expiration sweep. Optional [asOf] timestamp (unix + * seconds) lets the store pin the cutoff it actually used, so + * the projection's in-memory drop matches the store's on-disk + * drop exactly. When `null` the projection uses its own clock. + */ + data class Expired( + val asOf: Long? = null, + ) : DeleteRule +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt index 4e7c00bb4..3843fd8f0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt @@ -27,7 +27,9 @@ import com.vitorpamplona.quartz.nip01Core.core.isAddressable import com.vitorpamplona.quartz.nip01Core.core.isReplaceable import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.observable.DeleteRule import com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore +import com.vitorpamplona.quartz.nip01Core.store.observable.StoreEvent import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip40Expiration.expiration import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent @@ -36,7 +38,6 @@ import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job -import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -64,30 +65,35 @@ import kotlinx.coroutines.yield * - **Removal** (NIP-09 deletion, NIP-62 vanish, NIP-40 expiration) * drops the handle from the list. * - * The seed is materialised by querying the store once at start, after - * which the projection is driven entirely by [ObservableEventStore.events] - * and its own expiration ticker. The store is never re-queried on - * mutation, and the projection never asks the store to delete - * anything — it interprets incoming Nostr events itself: + * The seed is materialised by querying the store once at start, + * after which the projection is driven entirely by + * [ObservableEventStore.events]. Two kinds of mutation arrive on + * that stream: * - * - **NIP-01 supersession.** New replaceable / addressable events - * replace prior ones for the same `kind:pubkey[:dtag]`. The - * NIP-01 lexical-id tiebreaker (`new.id < old.id` when - * `created_at` ties) is honoured. - * - **NIP-09 deletions.** A [DeletionEvent] removes any matching - * handle owned by the same author (for GiftWrap, the recipient). - * Cross-author deletions are inert. - * - **NIP-62 right to vanish.** A [RequestToVanishEvent] whose - * `shouldVanishFrom([relay])` is true drops every handle from the - * same author with `created_at < vanish.created_at`. - * - **NIP-40 expiration.** Events with a past `expiration` tag are - * rejected at insert time. A periodic ticker drops slots whose - * expiration has just lapsed; collectors see the slot disappear. + * - [StoreEvent.Insert] — interpreted in-projection so a single + * arriving event can carry NIP-01 / NIP-09 / NIP-62 semantics: * - * That's the same set of rules the SQLite / FS stores enforce for - * durability. The duplication is by design — the store enforces them - * on disk so the file isn't corrupt; the projection enforces them in - * memory so the live view stays correct without a re-query per event. + * - **NIP-01 supersession.** New replaceable / addressable events + * replace prior ones for the same `kind:pubkey[:dtag]`. The + * NIP-01 lexical-id tiebreaker (`new.id < old.id` when + * `created_at` ties) is honoured. + * - **NIP-09 deletions.** A [DeletionEvent] removes any matching + * handle owned by the same author (for GiftWrap, the recipient). + * Cross-author deletions are inert. + * - **NIP-62 right to vanish.** A [RequestToVanishEvent] whose + * `shouldVanishFrom([relay])` is true drops every handle from + * the same author with `created_at < vanish.created_at`. + * - **NIP-40 expiration.** Events whose `expiration` tag has + * already lapsed at the moment they arrive are dropped before + * they ever enter [items]. + * + * - [StoreEvent.Delete] — emitted when a caller invokes + * `delete(filter)`, `delete(filters)`, or `deleteExpiredEvents()` + * on the [ObservableEventStore]. The projection drops every slot + * matching the rule using the same [Filter.match] / NIP-40 + * expiration semantics the store used. **There is no per-projection + * expiration ticker** — projections only drop expired events when + * the application calls `deleteExpiredEvents()` on the store. * * Limit handling: the initial query honours the filter `limit`, and * we trim to the same cap when an insert pushes the list over. We do @@ -96,29 +102,23 @@ import kotlinx.coroutines.yield * new arrives. That tradeoff matches what callers were already getting * from `LocalCache.observeEvents`. * - * Out-of-band store mutations — `delete(id)`, `delete(filter)`, - * `clearDB()`, the periodic `deleteExpiredEvents()` sweep — are not - * visible on [ObservableEventStore.events] and won't update an open - * projection. Re-open the projection (e.g. cancel and recreate the - * scope) to pick up an out-of-band change. - * * Ephemeral events (kinds `20000-29999`) reach the projection via * [ObservableEventStore.events] without ever being persisted; they * appear in [items] for as long as the projection is alive but never - * survive a re-seed. NIP-40 expiration applies to them too — if they - * carry an `expiration` tag, the per-projection ticker drops them - * when it lapses. + * survive a re-seed. They aren't covered by the store's + * `deleteExpiredEvents()` sweep (the DB never had them), so an + * ephemeral with an `expiration` tag will linger in the projection + * until it's superseded or until the projection is closed. * - * Lifecycle: the projection runs a collector + an expiration ticker - * inside [scope]. Cancel the scope (or call [close]) when the screen - * using the projection goes away. + * Lifecycle: the projection runs a single collector inside [scope]. + * Cancel the scope (or call [close]) when the screen using the + * projection goes away. */ class EventStoreProjection( private val store: ObservableEventStore, private val filters: List, private val relay: NormalizedRelayUrl?, scope: CoroutineScope, - private val expirationTickMs: Long = 30_000L, private val nowProvider: () -> Long = TimeUtils::now, ) : AutoCloseable { private val _items = MutableStateFlow>>(emptyList()) @@ -156,27 +156,17 @@ class EventStoreProjection( .onSubscription { seed() ready.complete(Unit) - }.collect { event -> apply(event) } - } - - private val expirationJob: Job = - scope.launch { - // Sleep first so the seed-time sweep covers the initial - // contents — see [seed]. - while (true) { - delay(expirationTickMs) - sweepExpired() - } + }.collect { storeEvent -> apply(storeEvent) } } private suspend fun seed() { val initial = store.query(filters) val now = nowProvider() for (event in initial) { - // The store should already exclude expired rows from the - // result, but it doesn't hurt to skip them here too — - // covers any FS / in-memory store that hasn't run a sweep - // recently. + // Stores don't filter expired events at query time, so we + // do it here — otherwise an expired event would briefly + // appear in [items] before the next deleteExpiredEvents() + // sweep clears it. if (isExpiredAt(event, now)) continue insertNew(event) yield() @@ -184,7 +174,14 @@ class EventStoreProjection( publish() } - private fun apply(event: Event) { + private fun apply(storeEvent: StoreEvent) { + when (storeEvent) { + is StoreEvent.Insert -> applyInsert(storeEvent.event) + is StoreEvent.Delete -> applyDelete(storeEvent.rule) + } + } + + private fun applyInsert(event: Event) { if (isExpiredAt(event, nowProvider())) return var changed = false @@ -207,6 +204,27 @@ class EventStoreProjection( if (changed) publish() } + private fun applyDelete(rule: DeleteRule) { + val targets = + when (rule) { + is DeleteRule.Filtered -> { + if (rule.filters.isEmpty()) return + byId.values.filter { slot -> rule.filters.any { it.match(slot.flow.value) } } + } + + is DeleteRule.Expired -> { + val cutoff = rule.asOf ?: nowProvider() + byId.values.filter { isExpiredAt(it.flow.value, cutoff) } + } + } + if (targets.isEmpty()) return + var changed = false + for (slot in targets) { + if (removeSlot(slot)) changed = true + } + if (changed) publish() + } + /** * Returns true if the event matches the filter and its arrival * caused membership to change (a fresh slot was added). Returns @@ -280,17 +298,6 @@ class EventStoreProjection( return changed } - private fun sweepExpired() { - val now = nowProvider() - val targets = byId.values.filter { isExpiredAt(it.flow.value, now) } - if (targets.isEmpty()) return - var changed = false - for (slot in targets) { - if (removeSlot(slot)) changed = true - } - if (changed) publish() - } - @Suppress("UNCHECKED_CAST") private fun insertNew(event: Event) { val slot = Slot(event as T) @@ -325,11 +332,10 @@ class EventStoreProjection( /** * Stop tracking changes and clear internal state. Idempotent. The * scope passed to the constructor keeps running; only this - * projection's collector + expiration jobs are cancelled. + * projection's collector job is cancelled. */ override fun close() { collectorJob.cancel() - expirationJob.cancel() ordered.clear() byId.clear() byStableKey.clear() diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt index 182847661..4e7382786 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt @@ -349,12 +349,12 @@ class EventStoreProjectionTest { } /** - * NIP-40 per-projection ticker: a slot whose `expiration` lapses - * after the projection has loaded should be dropped on the next - * tick, even though the store hasn't run its sweep yet. + * NIP-40 expiration drops slots only when the application calls + * `deleteExpiredEvents()` on the observable store — projections + * no longer run their own ticker. */ @Test - fun nip40ExpirationDroppedByTicker() = + fun nip40ExpirationDroppedOnStoreSweep() = runBlocking { val time = TimeUtils.now() val safe = signer.sign(TextNoteEvent.build("safe", createdAt = time) { expiration(time + 100) }) @@ -362,26 +362,59 @@ class EventStoreProjectionTest { observable.insert(safe) observable.insert(short) - // Drive the ticker frequently so the test doesn't sit idle. val projection = - EventStoreProjection( - observable, - listOf(Filter(kinds = listOf(TextNoteEvent.KIND))), - relay = null, - scope = scope, - expirationTickMs = 100, + observable.observe( + Filter(kinds = listOf(TextNoteEvent.KIND)), + store.relay, + scope, ) projection.ready.await() assertEquals(2, projection.items.value.size) - // Wait past the short expiration. + // Let the short expiration lapse, then ask the store to + // sweep — the projection drops the expired slot in + // response to the resulting StoreEvent.Delete(Expired). delay(2000) + observable.deleteExpiredEvents() - val after = projection.awaitItems(timeoutMs = 5_000) { it.size == 1 } + val after = projection.awaitItems { it.size == 1 } assertEquals(safe.id, after[0].value.id) projection.close() } + /** + * `delete(filter)` on the observable propagates to open + * projections: they drop every slot matching the filter using + * the same Filter.match logic the store would. + */ + @Test + fun deleteByFilterRemovesMatchingSlots() = + runBlocking { + val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) + val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) + val foreign = otherSigner.sign(TextNoteEvent.build("foreign", createdAt = 150)) + observable.insert(a) + observable.insert(b) + observable.insert(foreign) + + val projection = + observable.observe( + Filter(kinds = listOf(TextNoteEvent.KIND)), + store.relay, + scope, + ) + projection.ready.await() + assertEquals(3, projection.items.value.size) + + // Drop everything authored by `signer` — should leave + // only the foreign event. + observable.delete(Filter(authors = listOf(signer.pubKey))) + + val after = projection.awaitItems { it.size == 1 } + assertEquals(foreign.id, after[0].value.id) + projection.close() + } + @Test fun limitIsEnforcedOnInsertOverflow() = runBlocking { From f666fe20026e156a5a2dc991f960930bf130ab1b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 21:42:46 +0000 Subject: [PATCH 06/24] refactor(quartz): key projection's address index by Address instead of String MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames `byStableKey: HashMap` to `byAddress: HashMap` in EventStoreProjection. Address is a data class on every platform so equals/hashCode are derived from (kind, pubKeyHex, dTag) — same identity as the synthetic string key, but typed. `stableKey(event)` / `stableKey(kind, pubKeyHex, dTag)` become `addressOf(...)` returning `Address?`. Plain replaceables map to `Address(kind, pubKey, "")`, addressables to `Address(kind, pubKey, dTag)`, regular events to null. `limit` was already used at insertNew() to enforce the cap on inserts; left unchanged. 15/15 projection tests still pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../store/projection/EventStoreProjection.kt | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt index 3843fd8f0..bf6d8b7b4 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.quartz.nip01Core.store.projection +import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey @@ -128,10 +129,13 @@ class EventStoreProjection( private val byId = HashMap>() /** - * Slots keyed by `kind:pubkey:dtag` (or `kind:pubkey:` for plain - * replaceables) for in-place updates and supersession lookups. + * Slots keyed by replaceable / addressable address for in-place + * updates and supersession lookups. Plain replaceables (kind 0/3, + * 10000-19999) live under an [Address] with an empty `dTag`, + * matching how the SQLite store keys its `addressable_idx` / + * `replaceable_idx`. */ - private val byStableKey = HashMap>() + private val byAddress = HashMap>() /** * Sorted view of the same slots. The comparator uses each slot's @@ -233,9 +237,9 @@ class EventStoreProjection( */ @Suppress("UNCHECKED_CAST") private fun handleInsert(event: Event): Boolean { - val key = stableKey(event) - if (key != null) { - val existing = byStableKey[key] + val address = addressOf(event) + if (address != null) { + val existing = byAddress[address] if (existing != null) { if (!supersedes(event, existing.flow.value)) return false @@ -274,8 +278,8 @@ class EventStoreProjection( // with `created_at <= deletion.created_at`. for (addr in deletion.deleteAddresses()) { if (addr.pubKeyHex != deletion.pubKey) continue - val key = stableKey(addr.kind, addr.pubKeyHex, addr.dTag) ?: continue - val slot = byStableKey[key] ?: continue + val key = addressOf(addr.kind, addr.pubKeyHex, addr.dTag) ?: continue + val slot = byAddress[key] ?: continue if (slot.flow.value.createdAt <= deletion.createdAt) { if (removeSlot(slot)) changed = true } @@ -302,7 +306,7 @@ class EventStoreProjection( private fun insertNew(event: Event) { val slot = Slot(event as T) byId[event.id] = slot - stableKey(event)?.let { byStableKey[it] = slot } + addressOf(event)?.let { byAddress[it] = slot } ordered.add(slot) val cap = limit ?: return @@ -316,11 +320,11 @@ class EventStoreProjection( val removed = byId.remove(slot.flow.value.id) != null if (!removed) return false ordered.remove(slot) - stableKey(slot.flow.value)?.let { key -> - // Defensive: only clear the stable-key map if this slot + addressOf(slot.flow.value)?.let { address -> + // Defensive: only clear the address index if this slot // still owns it. Could be stale after an addressable rekey // raced with another insert. - if (byStableKey[key] === slot) byStableKey.remove(key) + if (byAddress[address] === slot) byAddress.remove(address) } return true } @@ -338,7 +342,7 @@ class EventStoreProjection( collectorJob.cancel() ordered.clear() byId.clear() - byStableKey.clear() + byAddress.clear() _items.value = emptyList() } @@ -359,19 +363,20 @@ class EventStoreProjection( companion object { /** - * The lookup key for replaceable / addressable supersession. - * `null` for regular events (which only collide on event id). + * The lookup [Address] for replaceable / addressable + * supersession. `null` for regular events (which only collide + * on event id). */ - private fun stableKey(event: Event): String? = stableKey(event.kind, event.pubKey, (event as? AddressableEvent)?.dTag()) + private fun addressOf(event: Event): Address? = addressOf(event.kind, event.pubKey, (event as? AddressableEvent)?.dTag()) - private fun stableKey( + private fun addressOf( kind: Int, pubKeyHex: HexKey, dTag: String?, - ): String? = + ): Address? = when { - kind.isAddressable() -> "$kind:$pubKeyHex:${dTag ?: ""}" - kind.isReplaceable() -> "$kind:$pubKeyHex:" + kind.isAddressable() -> Address(kind, pubKeyHex, dTag ?: "") + kind.isReplaceable() -> Address(kind, pubKeyHex, "") else -> null } From 544d26d9ecb3da3ba37d5cb3ea4548d892782e0d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Apr 2026 22:43:46 +0000 Subject: [PATCH 07/24] refactor(quartz): tidy projection package, flatten StoreEvent, per-filter limits Five changes wrapped together: 1. Move ObservableEventStore + StoreEvent from store/observable/ into store/projection/. One package owns the projection bus + projection itself; consumers import only `store.projection.*`. 2. Flatten DeleteRule into the StoreEvent sealed type: StoreEvent.Insert(event) StoreEvent.DeleteByFilter(filters) StoreEvent.DeleteExpired(asOf?) ObservableEventStore.delete(filter|filters) emits DeleteByFilter; deleteExpiredEvents() emits DeleteExpired with the pinned cutoff. 3. Use event.address() for AddressableEvent and event.isExpirationBefore(t) for NIP-40 checks; drop the local addressOf(kind, pubKey, dTag) and isExpiredAt helpers. Plain replaceables (kind 0/3, 10000-19999) still build Address(kind, pubKey, "") manually since they don't implement AddressableEvent. NIP-09 delete-by-address now looks up the projection's byAddress index with the deletion's Address directly (it's already an Address, no conversion needed). 4. Per-filter limit. Each filter retains its own capped TreeSet (sorted created_at DESC, id ASC). A slot is "live" iff at least one filter retains it. The projection's items is the deduped union, so when filter A and filter B match disjoint events the union can exceed any single filter's limit. New tests confirm both behaviours: per-filter eviction + union-larger-than-cap. 5. Drop the redundant + 1 in expiration checks; use isExpirationBefore directly with TimeUtils.now() for arrival-time guards. The sweep path subtracts 1 because the store's SQL uses strict `<` while the helper is `<=`. 17/17 projection tests + all other store tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../store/projection/EventStoreProjection.kt | 229 ++++++++++-------- .../ObservableEventStore.kt | 9 +- .../{observable => projection}/StoreEvent.kt | 46 ++-- .../projection/EventStoreProjectionTest.kt | 66 ++++- 4 files changed, 215 insertions(+), 135 deletions(-) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/{observable => projection}/ObservableEventStore.kt (95%) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/{observable => projection}/StoreEvent.kt (63%) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt index bf6d8b7b4..eb0987880 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt @@ -24,15 +24,11 @@ import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey -import com.vitorpamplona.quartz.nip01Core.core.isAddressable import com.vitorpamplona.quartz.nip01Core.core.isReplaceable import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.store.observable.DeleteRule -import com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore -import com.vitorpamplona.quartz.nip01Core.store.observable.StoreEvent import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent -import com.vitorpamplona.quartz.nip40Expiration.expiration +import com.vitorpamplona.quartz.nip40Expiration.isExpirationBefore import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.TimeUtils @@ -49,9 +45,8 @@ import kotlinx.coroutines.yield /** * A reactive projection over an [ObservableEventStore] for a fixed * set of [filters]. Each visible event is wrapped in a - * [MutableStateFlow] so - * the UI can collect three different kinds of change with the right - * granularity: + * [MutableStateFlow] so the UI can collect three different kinds of + * change with the right granularity: * * - **Membership** (events arriving or leaving) re-emits a brand new * [List] from [items]. The list reference is stable while membership @@ -63,17 +58,16 @@ import kotlinx.coroutines.yield * reshuffled when the new version has a later `created_at` — each * slot remembers the sort key it was inserted with, so updates feel * like pure value mutations. - * - **Removal** (NIP-09 deletion, NIP-62 vanish, NIP-40 expiration) - * drops the handle from the list. + * - **Removal** (NIP-09 deletion, NIP-62 vanish, NIP-40 expiration, + * `delete(filter)`) drops the handle from the list. * - * The seed is materialised by querying the store once at start, - * after which the projection is driven entirely by - * [ObservableEventStore.events]. Two kinds of mutation arrive on + * The seed is materialised by querying the store once at start, after + * which the projection is driven entirely by + * [ObservableEventStore.events]. Three kinds of mutation arrive on * that stream: * * - [StoreEvent.Insert] — interpreted in-projection so a single * arriving event can carry NIP-01 / NIP-09 / NIP-62 semantics: - * * - **NIP-01 supersession.** New replaceable / addressable events * replace prior ones for the same `kind:pubkey[:dtag]`. The * NIP-01 lexical-id tiebreaker (`new.id < old.id` when @@ -88,20 +82,23 @@ import kotlinx.coroutines.yield * already lapsed at the moment they arrive are dropped before * they ever enter [items]. * - * - [StoreEvent.Delete] — emitted when a caller invokes - * `delete(filter)`, `delete(filters)`, or `deleteExpiredEvents()` - * on the [ObservableEventStore]. The projection drops every slot - * matching the rule using the same [Filter.match] / NIP-40 - * expiration semantics the store used. **There is no per-projection + * - [StoreEvent.DeleteByFilter] — emitted on `delete(filter)` / + * `delete(filters)`. The projection drops every slot matching any + * of the rule's filters via [Filter.match]. + * + * - [StoreEvent.DeleteExpired] — emitted on `deleteExpiredEvents()`. + * The projection drops every slot whose `expiration` has lapsed at + * the cutoff the store pinned. **There is no per-projection * expiration ticker** — projections only drop expired events when * the application calls `deleteExpiredEvents()` on the store. * - * Limit handling: the initial query honours the filter `limit`, and - * we trim to the same cap when an insert pushes the list over. We do - * **not** refill from the store after a deletion — if a deletion - * leaves you under the limit, you stay under the limit until something - * new arrives. That tradeoff matches what callers were already getting - * from `LocalCache.observeEvents`. + * Limit handling is **per-filter**: each filter retains at most its + * own `limit` matches in a private capped set, sorted by created_at + * DESC + id ASC. The projection's [items] is the deduped union of + * those sets, so when filter A and filter B match disjoint events the + * union can be larger than any single filter's `limit`. We do not + * refill from the store after a deletion — if a removal leaves a + * filter under cap, it stays under cap until another match arrives. * * Ephemeral events (kinds `20000-29999`) reach the projection via * [ObservableEventStore.events] without ever being persisted; they @@ -138,13 +135,20 @@ class EventStoreProjection( private val byAddress = HashMap>() /** - * Sorted view of the same slots. The comparator uses each slot's - * frozen sort key (the seed event's `created_at` + `id`), so - * supersession in-place updates never move a slot inside this set. + * Per-filter capped sets. Each filter independently retains at most + * `filter.limit` matches; a slot is "live" (visible in [items]) + * iff it appears in at least one of these sets. Identity-keyed — + * [Filter] is `@Stable` but not a data class. */ - private val ordered = sortedSetOf(slotComparator()) + private val perFilter: Map>> = + filters.associateWith { sortedSetOf(slotComparator()) } - private val limit: Int? = filters.mapNotNull { it.limit }.maxOrNull() + /** + * Sorted union view of every "live" slot. Maintained alongside + * [perFilter] — a slot is added the first time any filter retains + * it and removed once no filter still holds it. + */ + private val ordered: java.util.SortedSet> = sortedSetOf(slotComparator()) /** Set when the seed has been written to [items], so callers can suspend until the projection is hot. */ val ready: CompletableDeferred = CompletableDeferred() @@ -171,8 +175,8 @@ class EventStoreProjection( // do it here — otherwise an expired event would briefly // appear in [items] before the next deleteExpiredEvents() // sweep clears it. - if (isExpiredAt(event, now)) continue - insertNew(event) + if (event.isExpirationBefore(now)) continue + applyInsert(event) yield() } publish() @@ -180,13 +184,23 @@ class EventStoreProjection( private fun apply(storeEvent: StoreEvent) { when (storeEvent) { - is StoreEvent.Insert -> applyInsert(storeEvent.event) - is StoreEvent.Delete -> applyDelete(storeEvent.rule) + is StoreEvent.Insert -> { + if (applyInsert(storeEvent.event)) publish() + } + + is StoreEvent.DeleteByFilter -> { + if (applyDeleteByFilter(storeEvent.filters)) publish() + } + + is StoreEvent.DeleteExpired -> { + val cutoff = storeEvent.asOf ?: nowProvider() + if (applyDeleteExpired(cutoff)) publish() + } } } - private fun applyInsert(event: Event) { - if (isExpiredAt(event, nowProvider())) return + private fun applyInsert(event: Event): Boolean { + if (event.isExpirationBefore(nowProvider())) return false var changed = false @@ -201,49 +215,54 @@ class EventStoreProjection( if (handleVanish(event)) changed = true } - if (filters.any { it.match(event) }) { - if (handleInsert(event)) changed = true - } + if (handleInsert(event)) changed = true - if (changed) publish() + return changed } - private fun applyDelete(rule: DeleteRule) { - val targets = - when (rule) { - is DeleteRule.Filtered -> { - if (rule.filters.isEmpty()) return - byId.values.filter { slot -> rule.filters.any { it.match(slot.flow.value) } } - } - - is DeleteRule.Expired -> { - val cutoff = rule.asOf ?: nowProvider() - byId.values.filter { isExpiredAt(it.flow.value, cutoff) } - } - } - if (targets.isEmpty()) return + private fun applyDeleteByFilter(rules: List): Boolean { + if (rules.isEmpty()) return false + val targets = byId.values.filter { slot -> rules.any { it.match(slot.flow.value) } } + if (targets.isEmpty()) return false var changed = false for (slot in targets) { if (removeSlot(slot)) changed = true } - if (changed) publish() + return changed + } + + private fun applyDeleteExpired(asOf: Long): Boolean { + // Store's sweep uses strict `<`; isExpirationBefore is `<=`, + // so subtract 1 to match. (Resolution is 1 second; the + // off-by-one in the rare equal-timestamp case lines up with + // the store's behaviour.) + val cutoff = asOf - 1 + val targets = byId.values.filter { it.flow.value.isExpirationBefore(cutoff) } + if (targets.isEmpty()) return false + var changed = false + for (slot in targets) { + if (removeSlot(slot)) changed = true + } + return changed } /** - * Returns true if the event matches the filter and its arrival - * caused membership to change (a fresh slot was added). Returns - * false when the arrival was an in-place supersession update or - * was rejected by the NIP-01 tiebreaker. + * Returns true if processing the event caused membership of the + * projection's [items] list to change. Returns false for in-place + * supersession updates, NIP-01 tiebreaker rejections, and arrivals + * that no filter matches. */ @Suppress("UNCHECKED_CAST") private fun handleInsert(event: Event): Boolean { val address = addressOf(event) + if (address != null) { val existing = byAddress[address] if (existing != null) { if (!supersedes(event, existing.flow.value)) return false - // Same address, new winner. Rekey byId and update the + // Same address, new winner. Rekey byId from the + // previous event id to the new one and update the // handle's value in place — list reference stays the // same; only the handle's collectors re-render. val previousId = existing.flow.value.id @@ -258,7 +277,34 @@ class EventStoreProjection( return false } - insertNew(event) + // Genuinely new slot. Offer it to every matching filter; if + // any retains it, the slot becomes live. + val slot = Slot(event as T) + var retained = false + for ((f, set) in perFilter) { + if (!f.match(event)) continue + set.add(slot) + retained = true + val cap = f.limit ?: continue + while (set.size > cap) { + val tail = set.last() + set.remove(tail) + if (tail === slot) { + // We were evicted by our own filter before any + // other filter got a chance to retain us — the + // remaining loop iterations may still pick us up. + retained = false + } else if (perFilter.values.none { it.contains(tail) }) { + // Tail no longer retained by any filter — drop it. + dropSlotIndexes(tail) + } + } + } + if (!retained) return false + + byId[event.id] = slot + if (address != null) byAddress[address] = slot + ordered.add(slot) return true } @@ -275,11 +321,11 @@ class EventStoreProjection( } // NIP-09: delete by address, only original author, only events - // with `created_at <= deletion.created_at`. + // with `created_at <= deletion.created_at`. `addr` is already + // an [Address] — same equals/hashCode as our index keys. for (addr in deletion.deleteAddresses()) { if (addr.pubKeyHex != deletion.pubKey) continue - val key = addressOf(addr.kind, addr.pubKeyHex, addr.dTag) ?: continue - val slot = byAddress[key] ?: continue + val slot = byAddress[addr] ?: continue if (slot.flow.value.createdAt <= deletion.createdAt) { if (removeSlot(slot)) changed = true } @@ -302,33 +348,33 @@ class EventStoreProjection( return changed } - @Suppress("UNCHECKED_CAST") - private fun insertNew(event: Event) { - val slot = Slot(event as T) - byId[event.id] = slot - addressOf(event)?.let { byAddress[it] = slot } - ordered.add(slot) - - val cap = limit ?: return - while (ordered.size > cap) { - val tail = ordered.last() - removeSlot(tail) - } - } - + /** Drop a live slot from every index AND from each per-filter set. */ private fun removeSlot(slot: Slot): Boolean { val removed = byId.remove(slot.flow.value.id) != null if (!removed) return false ordered.remove(slot) addressOf(slot.flow.value)?.let { address -> // Defensive: only clear the address index if this slot - // still owns it. Could be stale after an addressable rekey - // raced with another insert. + // still owns it. if (byAddress[address] === slot) byAddress.remove(address) } + for (set in perFilter.values) set.remove(slot) return true } + /** + * Drop a slot from byId / byAddress / ordered without touching the + * per-filter sets. Used when the per-filter eviction loop already + * owns the bookkeeping for those sets. + */ + private fun dropSlotIndexes(slot: Slot) { + byId.remove(slot.flow.value.id) + ordered.remove(slot) + addressOf(slot.flow.value)?.let { address -> + if (byAddress[address] === slot) byAddress.remove(address) + } + } + private fun publish() { _items.value = ordered.map { it.flow } } @@ -343,6 +389,7 @@ class EventStoreProjection( ordered.clear() byId.clear() byAddress.clear() + for (set in perFilter.values) set.clear() _items.value = emptyList() } @@ -351,7 +398,7 @@ class EventStoreProjection( * one of these for as long as it survives. The sort key is frozen * at construction time — supersession in-place updates rewrite * `flow.value` but never the sort key, so the ordering inside - * [ordered] is stable across updates. + * [ordered] / [perFilter] is stable across updates. */ private class Slot( initial: T, @@ -367,16 +414,10 @@ class EventStoreProjection( * supersession. `null` for regular events (which only collide * on event id). */ - private fun addressOf(event: Event): Address? = addressOf(event.kind, event.pubKey, (event as? AddressableEvent)?.dTag()) - - private fun addressOf( - kind: Int, - pubKeyHex: HexKey, - dTag: String?, - ): Address? = + private fun addressOf(event: Event): Address? = when { - kind.isAddressable() -> Address(kind, pubKeyHex, dTag ?: "") - kind.isReplaceable() -> Address(kind, pubKeyHex, "") + event is AddressableEvent -> event.address() + event.kind.isReplaceable() -> Address(event.kind, event.pubKey, "") else -> null } @@ -403,14 +444,6 @@ class EventStoreProjection( */ private fun ownerPubKey(event: Event): HexKey = (event as? GiftWrapEvent)?.recipientPubKey() ?: event.pubKey - private fun isExpiredAt( - event: Event, - now: Long, - ): Boolean { - val exp = event.expiration() ?: return false - return exp <= now - } - /** * created_at DESC, id ASC. The keys are snapshots taken at * insertion time, so the ordering of a slot never changes diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt similarity index 95% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt index 3f66839d2..d9c4699f6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt @@ -18,14 +18,13 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.store.observable +package com.vitorpamplona.quartz.nip01Core.store.projection import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.isEphemeral import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore -import com.vitorpamplona.quartz.nip01Core.store.projection.EventStoreProjection import com.vitorpamplona.quartz.nip40Expiration.isExpired import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.CoroutineScope @@ -144,12 +143,12 @@ class ObservableEventStore( override suspend fun delete(filter: Filter) { inner.delete(filter) - _events.emit(StoreEvent.Delete(DeleteRule.Filtered(listOf(filter)))) + _events.emit(StoreEvent.DeleteByFilter(listOf(filter))) } override suspend fun delete(filters: List) { inner.delete(filters) - _events.emit(StoreEvent.Delete(DeleteRule.Filtered(filters))) + _events.emit(StoreEvent.DeleteByFilter(filters)) } override suspend fun deleteExpiredEvents() { @@ -160,7 +159,7 @@ class ObservableEventStore( // own clock when the event is processed. val asOf = TimeUtils.now() inner.deleteExpiredEvents() - _events.emit(StoreEvent.Delete(DeleteRule.Expired(asOf))) + _events.emit(StoreEvent.DeleteExpired(asOf)) } /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/StoreEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreEvent.kt similarity index 63% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/StoreEvent.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreEvent.kt index 83d46d46a..e0b66ba46 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/observable/StoreEvent.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreEvent.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.store.observable +package com.vitorpamplona.quartz.nip01Core.store.projection import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter @@ -32,40 +32,26 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter * layer (persistable or ephemeral). Carries the event itself so * the projection can run its NIP-01 / NIP-09 / NIP-62 * interpretation. - * - [Delete] is emitted for every out-of-band removal — manual - * `delete(filter)` / `delete(filters)` calls and the periodic - * `deleteExpiredEvents()` sweep. Carries the rule the store used - * so the projection can apply the same selection in memory - * without re-querying. + * - [DeleteByFilter] is emitted for every `delete(filter)` / + * `delete(filters)` call on the observable. Carries the same + * filters the store used so projections can apply + * [Filter.match] in memory and drop the matching slots without + * re-querying. + * - [DeleteExpired] is emitted for every `deleteExpiredEvents()` + * sweep. The optional [DeleteExpired.asOf] cutoff lets the store + * pin the timestamp it actually used, so the projection drops + * exactly the events the store dropped. */ sealed interface StoreEvent { data class Insert( val event: Event, ) : StoreEvent - data class Delete( - val rule: DeleteRule, + data class DeleteByFilter( + val filters: List, + ) : StoreEvent + + data class DeleteExpired( + val asOf: Long? = null, ) : StoreEvent } - -/** - * Selection rule for a [StoreEvent.Delete]. [Filtered] mirrors the - * `delete(filter)` / `delete(filters)` API and OR-combines the - * filters; [Expired] mirrors `deleteExpiredEvents()` and removes - * everything whose NIP-40 expiration has lapsed. - */ -sealed interface DeleteRule { - data class Filtered( - val filters: List, - ) : DeleteRule - - /** - * NIP-40 expiration sweep. Optional [asOf] timestamp (unix - * seconds) lets the store pin the cutoff it actually used, so - * the projection's in-memory drop matches the store's on-disk - * drop exactly. When `null` the projection uses its own clock. - */ - data class Expired( - val asOf: Long? = null, - ) : DeleteRule -} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt index 4e7382786..a5811f220 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt @@ -25,7 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync -import com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore +import com.vitorpamplona.quartz.nip01Core.store.projection.ObservableEventStore import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent @@ -442,6 +442,68 @@ class EventStoreProjectionTest { projection.close() } + /** + * Per-filter limit: filter A caps at 2, filter B caps at 2, but + * the events they match are disjoint, so the projection's union + * is 4 (larger than any single filter's cap). + */ + @Test + fun perFilterLimitUnionExceedsSingleLimit() = + runBlocking { + val authorA = NostrSignerSync() + val authorB = NostrSignerSync() + val a1 = authorA.sign(TextNoteEvent.build("a1", createdAt = 100)) + val a2 = authorA.sign(TextNoteEvent.build("a2", createdAt = 200)) + val b1 = authorB.sign(TextNoteEvent.build("b1", createdAt = 110)) + val b2 = authorB.sign(TextNoteEvent.build("b2", createdAt = 210)) + observable.insert(a1) + observable.insert(a2) + observable.insert(b1) + observable.insert(b2) + + val filterA = Filter(kinds = listOf(TextNoteEvent.KIND), authors = listOf(authorA.pubKey), limit = 2) + val filterB = Filter(kinds = listOf(TextNoteEvent.KIND), authors = listOf(authorB.pubKey), limit = 2) + val projection = + observable.observe( + listOf(filterA, filterB), + store.relay, + scope, + ) + projection.ready.await() + assertEquals(4, projection.items.value.size, "per-filter caps don't dedupe union") + projection.close() + } + + /** + * Each filter's cap is enforced on its own retained set: filter A + * keeps only its 2 newest matches even when more arrive. + */ + @Test + fun perFilterLimitEvictsOldestWithinFilter() = + runBlocking { + val a1 = signer.sign(TextNoteEvent.build("a1", createdAt = 100)) + val a2 = signer.sign(TextNoteEvent.build("a2", createdAt = 200)) + val a3 = signer.sign(TextNoteEvent.build("a3", createdAt = 300)) + observable.insert(a1) + observable.insert(a2) + + val projection = + observable.observe( + Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), + store.relay, + scope, + ) + projection.ready.await() + assertEquals(2, projection.items.value.size) + + observable.insert(a3) + val after = projection.awaitItems { it[0].value.id == a3.id } + assertEquals(2, after.size) + assertEquals(a3.id, after[0].value.id) + assertEquals(a2.id, after[1].value.id) + projection.close() + } + @Test fun closeStopsListening() = runBlocking { @@ -461,7 +523,7 @@ class EventStoreProjectionTest { * Ephemeral events (kinds `20000-29999`) are never persisted, so * the inner SQLite store silently drops them. The * `ObservableEventStore` wrapper still routes them onto its - * [events][com.vitorpamplona.quartz.nip01Core.store.observable.ObservableEventStore.events] + * [events][com.vitorpamplona.quartz.nip01Core.store.projection.ObservableEventStore.events] * flow, so an open projection sees them while it's alive. They * vanish from any future seed because the DB never had them. */ From ae6343cd73530db039bb3c9e2dc29b45d2b3a536 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 00:03:03 +0000 Subject: [PATCH 08/24] refactor(quartz): simplify EventStoreProjection (-50 lines, -1 bug) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the projection found one real bug and four redundancies: Bug fix: handleInsert tracked `retained` inline as it walked filters, flipping it to false when the slot self-evicted from a filter's cap. But if filter A retained the slot (cap=null or with room) and filter B later cap-evicted it, retained ended up false even though the slot was still in A's set — leaking a slot into a per-filter set with no byId/byAddress entry. Fix: compute retention after the loop with `perFilter.values.any { it.contains(slot) }`. Simplifications: - Drop the redundant `ordered: SortedSet` field. It was only read by publish(); it's the deduped sorted union of perFilter values. Compute it lazily at publish time instead. Removes the field, all the ordered.add/ordered.remove calls, and one helper. - Collapse `dropSlotIndexes` and `removeSlot` paths into `removeIndexes` (id+address) + `removeSlot` (also clears per-filter sets). The eviction loop calls removeIndexes; everywhere else calls removeSlot. - Drop the defensive `byAddress[address] === slot` identity check — with byId as the primary index, removeSlot is only called once per slot, and rekey only ever moves byId, never byAddress. - Pull the iterate-and-drop pattern into a single `dropWhere(predicate)` inline helper. handleVanish, applyDeleteByFilter, and applyDeleteExpired all collapse to one-liners. - Apply now returns Boolean from each branch and publish() runs once at the end, instead of three separate `if-publish` blocks. Net: 233 → 142 lines. Same behaviour, plus the retained bug fix. 17/17 projection tests + all other store tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../store/projection/EventStoreProjection.kt | 233 +++++++----------- 1 file changed, 91 insertions(+), 142 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt index eb0987880..0b20f919d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt @@ -125,31 +125,18 @@ class EventStoreProjection( /** Slots keyed by the *current* event id. Re-keyed when a replaceable / addressable handle takes a new version. */ private val byId = HashMap>() - /** - * Slots keyed by replaceable / addressable address for in-place - * updates and supersession lookups. Plain replaceables (kind 0/3, - * 10000-19999) live under an [Address] with an empty `dTag`, - * matching how the SQLite store keys its `addressable_idx` / - * `replaceable_idx`. - */ + /** Slots keyed by replaceable / addressable [Address] for in-place updates and supersession lookups. */ private val byAddress = HashMap>() /** * Per-filter capped sets. Each filter independently retains at most - * `filter.limit` matches; a slot is "live" (visible in [items]) - * iff it appears in at least one of these sets. Identity-keyed — - * [Filter] is `@Stable` but not a data class. + * `filter.limit` matches; a slot is "live" iff it appears in at + * least one of these sets. Identity-keyed — [Filter] is `@Stable` + * but not a data class. */ private val perFilter: Map>> = filters.associateWith { sortedSetOf(slotComparator()) } - /** - * Sorted union view of every "live" slot. Maintained alongside - * [perFilter] — a slot is added the first time any filter retains - * it and removed once no filter still holds it. - */ - private val ordered: java.util.SortedSet> = sortedSetOf(slotComparator()) - /** Set when the seed has been written to [items], so callers can suspend until the projection is hot. */ val ready: CompletableDeferred = CompletableDeferred() @@ -157,9 +144,9 @@ class EventStoreProjection( scope.launch { // `onSubscription` runs after the SharedFlow subscription is // active but before we pull any events — the buffer absorbs - // any emissions that arrive while we seed, and we then drain - // them as the collect proceeds. Doing the seed inside - // `collect { }` instead would race with concurrent inserts. + // emissions arriving during seed and drains them once collect + // proceeds. Doing the seed inside `collect { }` would race + // with concurrent inserts. store.events .onSubscription { seed() @@ -168,13 +155,11 @@ class EventStoreProjection( } private suspend fun seed() { - val initial = store.query(filters) val now = nowProvider() - for (event in initial) { - // Stores don't filter expired events at query time, so we - // do it here — otherwise an expired event would briefly - // appear in [items] before the next deleteExpiredEvents() - // sweep clears it. + for (event in store.query(filters)) { + // Stores don't filter expired rows at query time, so do it + // here — otherwise an expired event would briefly appear + // in [items] before the next deleteExpiredEvents() sweep. if (event.isExpirationBefore(now)) continue applyInsert(event) yield() @@ -183,20 +168,24 @@ class EventStoreProjection( } private fun apply(storeEvent: StoreEvent) { - when (storeEvent) { - is StoreEvent.Insert -> { - if (applyInsert(storeEvent.event)) publish() - } + val changed = + when (storeEvent) { + is StoreEvent.Insert -> { + applyInsert(storeEvent.event) + } - is StoreEvent.DeleteByFilter -> { - if (applyDeleteByFilter(storeEvent.filters)) publish() - } + is StoreEvent.DeleteByFilter -> { + dropWhere { ev -> storeEvent.filters.any { it.match(ev) } } + } - is StoreEvent.DeleteExpired -> { - val cutoff = storeEvent.asOf ?: nowProvider() - if (applyDeleteExpired(cutoff)) publish() + // Store's sweep uses strict `<`; isExpirationBefore is + // `<=`, so subtract 1 to match. + is StoreEvent.DeleteExpired -> { + val cutoff = (storeEvent.asOf ?: nowProvider()) - 1 + dropWhere { it.isExpirationBefore(cutoff) } + } } - } + if (changed) publish() } private fun applyInsert(event: Event): Boolean { @@ -204,15 +193,14 @@ class EventStoreProjection( var changed = false - // Apply NIP-09 / NIP-62 side effects of the event before we - // consider matching the event itself against the filter — a - // deletion event that arrives at the same instant as a - // matching event still removes its targets. + // NIP-09 / NIP-62 side effects come first — a deletion event + // that arrives at the same instant as a matching event still + // removes its targets. if (event is DeletionEvent) { if (handleDeletion(event)) changed = true } if (event is RequestToVanishEvent && event.shouldVanishFrom(relay)) { - if (handleVanish(event)) changed = true + if (dropWhere { ev -> ownerOf(ev) == event.pubKey && ev.createdAt < event.createdAt }) changed = true } if (handleInsert(event)) changed = true @@ -220,37 +208,11 @@ class EventStoreProjection( return changed } - private fun applyDeleteByFilter(rules: List): Boolean { - if (rules.isEmpty()) return false - val targets = byId.values.filter { slot -> rules.any { it.match(slot.flow.value) } } - if (targets.isEmpty()) return false - var changed = false - for (slot in targets) { - if (removeSlot(slot)) changed = true - } - return changed - } - - private fun applyDeleteExpired(asOf: Long): Boolean { - // Store's sweep uses strict `<`; isExpirationBefore is `<=`, - // so subtract 1 to match. (Resolution is 1 second; the - // off-by-one in the rare equal-timestamp case lines up with - // the store's behaviour.) - val cutoff = asOf - 1 - val targets = byId.values.filter { it.flow.value.isExpirationBefore(cutoff) } - if (targets.isEmpty()) return false - var changed = false - for (slot in targets) { - if (removeSlot(slot)) changed = true - } - return changed - } - /** - * Returns true if processing the event caused membership of the - * projection's [items] list to change. Returns false for in-place - * supersession updates, NIP-01 tiebreaker rejections, and arrivals - * that no filter matches. + * Returns true if processing the event caused the projection's + * membership to change. Returns false for in-place supersession + * updates, NIP-01 tiebreaker rejections, and arrivals that no + * filter matches. */ @Suppress("UNCHECKED_CAST") private fun handleInsert(event: Event): Boolean { @@ -278,105 +240,98 @@ class EventStoreProjection( } // Genuinely new slot. Offer it to every matching filter; if - // any retains it, the slot becomes live. + // any filter still holds it after cap-eviction, the slot + // becomes live and gets indexed. + var membershipChanged = false val slot = Slot(event as T) - var retained = false for ((f, set) in perFilter) { if (!f.match(event)) continue set.add(slot) - retained = true val cap = f.limit ?: continue while (set.size > cap) { val tail = set.last() set.remove(tail) - if (tail === slot) { - // We were evicted by our own filter before any - // other filter got a chance to retain us — the - // remaining loop iterations may still pick us up. - retained = false - } else if (perFilter.values.none { it.contains(tail) }) { - // Tail no longer retained by any filter — drop it. - dropSlotIndexes(tail) + if (tail !== slot && perFilter.values.none { it.contains(tail) }) { + // Tail no longer retained by any filter — fully drop. + if (removeIndexes(tail)) membershipChanged = true } } } - if (!retained) return false - byId[event.id] = slot - if (address != null) byAddress[address] = slot - ordered.add(slot) - return true + // The slot survived cap-eviction in at least one filter, so it + // belongs in the indexes. Otherwise nothing was indexed and + // the only membership effect is whatever evictions happened + // along the way. + if (perFilter.values.any { it.contains(slot) }) { + byId[event.id] = slot + if (address != null) byAddress[address] = slot + membershipChanged = true + } + return membershipChanged } private fun handleDeletion(deletion: DeletionEvent): Boolean { var changed = false - // NIP-09: delete by id, but only if the deletion's author owns - // the target. For GiftWrap, the owner is the p-tag recipient. + // NIP-09 by id, only if the deletion's author owns the target. + // For GiftWrap, the owner is the p-tag recipient. for (id in deletion.deleteEventIds()) { val slot = byId[id] ?: continue - val ev = slot.flow.value - val owner = ownerPubKey(ev) - if (owner == deletion.pubKey && removeSlot(slot)) changed = true + if (ownerOf(slot.flow.value) == deletion.pubKey && removeSlot(slot)) changed = true } - // NIP-09: delete by address, only original author, only events - // with `created_at <= deletion.created_at`. `addr` is already - // an [Address] — same equals/hashCode as our index keys. + // NIP-09 by address, only original author, only events with + // `created_at <= deletion.created_at`. `addr` is already an + // [Address] — same equals/hashCode as our index keys. for (addr in deletion.deleteAddresses()) { if (addr.pubKeyHex != deletion.pubKey) continue val slot = byAddress[addr] ?: continue - if (slot.flow.value.createdAt <= deletion.createdAt) { - if (removeSlot(slot)) changed = true - } + if (slot.flow.value.createdAt <= deletion.createdAt && removeSlot(slot)) changed = true } return changed } - private fun handleVanish(vanish: RequestToVanishEvent): Boolean { + /** Drop every live slot whose current event matches [predicate]. */ + private inline fun dropWhere(predicate: (Event) -> Boolean): Boolean { + // Snapshot before mutating — removeSlot mutates byId. + val targets = byId.values.filter { predicate(it.flow.value) } var changed = false - // Snapshot first because removeSlot mutates byId. - val targets = - byId.values.filter { - val ev = it.flow.value - ownerPubKey(ev) == vanish.pubKey && ev.createdAt < vanish.createdAt - } for (slot in targets) { if (removeSlot(slot)) changed = true } return changed } - /** Drop a live slot from every index AND from each per-filter set. */ + /** Remove a live slot from every index AND from each per-filter set. */ private fun removeSlot(slot: Slot): Boolean { - val removed = byId.remove(slot.flow.value.id) != null - if (!removed) return false - ordered.remove(slot) - addressOf(slot.flow.value)?.let { address -> - // Defensive: only clear the address index if this slot - // still owns it. - if (byAddress[address] === slot) byAddress.remove(address) - } for (set in perFilter.values) set.remove(slot) - return true + return removeIndexes(slot) } /** - * Drop a slot from byId / byAddress / ordered without touching the - * per-filter sets. Used when the per-filter eviction loop already - * owns the bookkeeping for those sets. + * Remove a slot from [byId] / [byAddress] without touching the + * per-filter sets. Used by the per-filter eviction loop, which + * already owns the bookkeeping for those. */ - private fun dropSlotIndexes(slot: Slot) { - byId.remove(slot.flow.value.id) - ordered.remove(slot) - addressOf(slot.flow.value)?.let { address -> - if (byAddress[address] === slot) byAddress.remove(address) - } + private fun removeIndexes(slot: Slot): Boolean { + val removed = byId.remove(slot.flow.value.id) != null + if (!removed) return false + addressOf(slot.flow.value)?.let(byAddress::remove) + return true } private fun publish() { - _items.value = ordered.map { it.flow } + // Lazily compute the deduped sorted union from per-filter + // sets. Cheaper than maintaining a separate `ordered` field + // alongside every insert / remove. + if (byId.isEmpty()) { + _items.value = emptyList() + return + } + val union = sortedSetOf(slotComparator()) + for (set in perFilter.values) union.addAll(set) + _items.value = union.map { it.flow } } /** @@ -386,7 +341,6 @@ class EventStoreProjection( */ override fun close() { collectorJob.cancel() - ordered.clear() byId.clear() byAddress.clear() for (set in perFilter.values) set.clear() @@ -397,8 +351,8 @@ class EventStoreProjection( * Internal slot. Each event added to the projection lives inside * one of these for as long as it survives. The sort key is frozen * at construction time — supersession in-place updates rewrite - * `flow.value` but never the sort key, so the ordering inside - * [ordered] / [perFilter] is stable across updates. + * `flow.value` but never the sort key, so the position inside + * each [perFilter] set is stable across updates. */ private class Slot( initial: T, @@ -409,11 +363,7 @@ class EventStoreProjection( } companion object { - /** - * The lookup [Address] for replaceable / addressable - * supersession. `null` for regular events (which only collide - * on event id). - */ + /** [Address] for replaceable / addressable supersession; `null` for regular events. */ private fun addressOf(event: Event): Address? = when { event is AddressableEvent -> event.address() @@ -422,10 +372,9 @@ class EventStoreProjection( } /** - * NIP-01 supersession tiebreaker. The new event wins iff: - * - its `created_at` is strictly greater, OR - * - the timestamps tie and its `id` is lexically smaller. - * Otherwise the existing slot keeps its place. + * NIP-01 supersession tiebreaker. The new event wins iff its + * `created_at` is strictly greater, or the timestamps tie and + * its `id` is lexically smaller. */ private fun supersedes( new: Event, @@ -442,13 +391,13 @@ class EventStoreProjection( * NIP-62 vanish target). For GiftWrap the owner is the p-tag * recipient; for everything else it's `event.pubKey`. */ - private fun ownerPubKey(event: Event): HexKey = (event as? GiftWrapEvent)?.recipientPubKey() ?: event.pubKey + private fun ownerOf(event: Event): HexKey = (event as? GiftWrapEvent)?.recipientPubKey() ?: event.pubKey /** * created_at DESC, id ASC. The keys are snapshots taken at - * insertion time, so the ordering of a slot never changes - * after it joins the set. Distinct events have distinct ids, - * so no third-key tiebreak is needed. + * insertion time, so a slot's position never changes after it + * joins a set. Distinct events have distinct ids, so no third + * tiebreak is needed. */ private fun slotComparator(): Comparator> = Comparator { a, b -> From e909866d26dd95774c512f0a9f11de8595be4498 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 13:27:33 +0000 Subject: [PATCH 09/24] feat(quartz): add EventInterner so deserialized events share canonical instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Process-wide interner that canonicalises Event instances by id — whenever the same event id is decoded twice (relay duplicates, projection seeds, FS re-reads, etc.), every consumer sees the same object reference. Backed by weak references so entries vanish once no live consumer (projection slot, UI state) holds the event. Shape: - expect class EventInterner in nip01Core/core/, with a Default process-wide instance. - jvmAndroid actual: ConcurrentHashMap> pre-sized to 5_000 (load factor 0.75). On-access cleanup atomically removes dead entries via map.remove(key, ref). intern() uses putIfAbsent + retry-on-dead-canonical to be race-safe. - apple / linux actuals: passthrough (no canonicalisation). Kotlin/ Native has weak refs but no built-in concurrent map; rather than ship a half-baked impl on Apple targets we skip canonicalisation there. Can be revisited if iOS wants the memory savings. Wire-up: - Event.fromJson now routes through EventInterner.Default.intern, so every deserialised event becomes canonical for free. In-process events (signer.sign(...) etc.) aren't auto-interned — the canonicaliser pays off for events that re-occur from multiple sources, which signed-locally events don't. Tests (jvmTest): - internReturnsFirstInstance / internCollapsesDuplicates: identity is preserved across equivalent decodes. - getReturnsLiveEntry / getReturnsNullForUnknownId. - getEvictsDeadEntries: weak ref cleared by GC, on-access cleanup drops the entry. - draftChurnDoesNotLeak: 100 distinct drafts churn through the interner with no strong refs; map shrinks back to ~0 after GC. - defaultInstanceIsShared: the global Default interner is process-wide. - 7/7 pass; 240/240 store + projection tests still green. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../nip01Core/core/EventInterner.apple.kt | 42 ++++++ .../quartz/nip01Core/core/Event.kt | 2 +- .../quartz/nip01Core/core/EventInterner.kt | 65 +++++++++ .../core/EventInterner.jvmAndroid.kt | 76 ++++++++++ .../nip01Core/core/EventInternerTest.kt | 136 ++++++++++++++++++ .../nip01Core/core/EventInterner.linux.kt | 36 +++++ 6 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.apple.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.kt create mode 100644 quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.jvmAndroid.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInternerTest.kt create mode 100644 quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.linux.kt diff --git a/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.apple.kt b/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.apple.kt new file mode 100644 index 000000000..4dc306f84 --- /dev/null +++ b/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.apple.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.core + +/** + * Apple actual: passthrough. Kotlin/Native has weak refs but no + * built-in concurrent map; rather than ship a half-baked impl, we + * skip canonicalisation entirely on Apple targets and let + * [Event.fromJson] return whatever the deserializer produced. Can + * be revisited if Amethyst's iOS port wants the memory savings. + */ +actual class EventInterner { + actual fun intern(event: Event): Event = event + + actual fun get(id: HexKey): Event? = null + + actual fun size(): Int = 0 + + actual fun clear() {} + + actual companion object { + actual val Default: EventInterner = EventInterner() + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/Event.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/Event.kt index 906773852..576fe6813 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/Event.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/Event.kt @@ -67,7 +67,7 @@ open class Event( fun toJson(): String = OptimizedJsonMapper.toJson(this) companion object { - fun fromJson(json: String): Event = OptimizedJsonMapper.fromJson(json) + fun fromJson(json: String): Event = EventInterner.Default.intern(OptimizedJsonMapper.fromJson(json)) fun fromJsonOrNull(json: String) = try { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.kt new file mode 100644 index 000000000..c0be04c21 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.core + +/** + * Process-wide interner that canonicalises [Event] instances by id so + * every consumer (relay client, store deserialization, projections, + * tests) sees the same object reference for the same event id. + * + * Backed by weak references — entries vanish when no live consumer + * holds the event, so the cache only grows as long as projections / + * UI state actually need the events. Sized for ~5000 hot events; the + * underlying map resizes if usage exceeds that. + * + * Use [intern] on every event arrival path. The first occurrence wins + * and becomes canonical; subsequent equivalent decodes return that + * canonical instance. + * + * Platforms without weak references fall back to a passthrough that + * returns [event] unchanged — no canonicalisation, but no leaks + * either. + */ +expect class EventInterner() { + /** + * Returns the canonical [Event] for [event]'s id. If a live + * canonical instance already exists for this id, returns it; + * otherwise stores [event] as the new canonical and returns it. + * + * Equivalence is by event id only — callers must trust the id + * was content-derived (signed events satisfy this). + */ + fun intern(event: Event): Event + + /** Returns the canonical [Event] for [id] if one is live, else null. */ + fun get(id: HexKey): Event? + + /** Number of map entries (including dead weak refs not yet cleaned). */ + fun size(): Int + + /** Drop every entry. Mostly useful for tests. */ + fun clear() + + companion object { + /** Process-wide default interner used by [Event.fromJson]. */ + val Default: EventInterner + } +} diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.jvmAndroid.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.jvmAndroid.kt new file mode 100644 index 000000000..f728608ac --- /dev/null +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.jvmAndroid.kt @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.core + +import java.lang.ref.WeakReference +import java.util.concurrent.ConcurrentHashMap + +/** + * JVM/Android actual: ConcurrentHashMap of WeakReference, pre-sized + * for the expected hot working set. Dead entries are reclaimed + * lazily on access (see [intern] and [get]) — when a weak ref's + * referent has been GC'd, the map slot is removed atomically and + * replaced if a fresh event takes its place. + */ +actual class EventInterner { + private val cache = ConcurrentHashMap>(INITIAL_CAPACITY, LOAD_FACTOR) + + actual fun intern(event: Event): Event { + // Fast path: existing canonical is still alive. + cache[event.id]?.get()?.let { return it } + + // Race-safe install: putIfAbsent ensures only one writer + // wins. If another thread beat us, return whatever they put + // (resolving the rare case where their ref was already GC'd + // by retrying through the slow path). + while (true) { + val ref = WeakReference(event) + val existing = cache.putIfAbsent(event.id, ref) + if (existing == null) return event + val canonical = existing.get() + if (canonical != null) return canonical + // Existing entry's referent was GC'd; drop it and retry. + cache.remove(event.id, existing) + } + } + + actual fun get(id: HexKey): Event? { + val ref = cache[id] ?: return null + val event = ref.get() + if (event != null) return event + // Self-cleaning: drop the dead entry so the map doesn't bloat. + cache.remove(id, ref) + return null + } + + actual fun size(): Int = cache.size + + actual fun clear() { + cache.clear() + } + + actual companion object { + actual val Default: EventInterner = EventInterner() + + private const val INITIAL_CAPACITY = 5_000 + private const val LOAD_FACTOR = 0.75f + } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInternerTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInternerTest.kt new file mode 100644 index 000000000..f55f79cd8 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInternerTest.kt @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.core + +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.utils.Secp256k1Instance +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertSame +import kotlin.test.assertTrue + +class EventInternerTest { + private val signer = NostrSignerSync() + private lateinit var interner: EventInterner + + @BeforeTest + fun setUp() { + Secp256k1Instance + interner = EventInterner() + } + + @AfterTest + fun tearDown() { + interner.clear() + } + + @Test + fun internReturnsFirstInstance() { + val original = signer.sign(TextNoteEvent.build("hello", createdAt = 100)) + val canonical = interner.intern(original) + assertSame(original, canonical) + assertEquals(1, interner.size()) + } + + @Test + fun internCollapsesDuplicates() { + val a = signer.sign(TextNoteEvent.build("hello", createdAt = 100)) + // Round-trip through OptimizedJsonMapper directly (NOT + // Event.fromJson, which would already intern via Default) so + // we get a non-identical-but-equal event for our isolated + // interner to deduplicate. + val b = OptimizedJsonMapper.fromJson(a.toJson()) as TextNoteEvent + assertEquals(a.id, b.id) + + val firstCanonical = interner.intern(a) + val secondCanonical = interner.intern(b) + assertSame(firstCanonical, secondCanonical) + assertSame(a, secondCanonical) + assertEquals(1, interner.size()) + } + + @Test + fun getReturnsLiveEntry() { + val a = signer.sign(TextNoteEvent.build("hello", createdAt = 100)) + interner.intern(a) + assertSame(a, interner.get(a.id)) + } + + @Test + fun getReturnsNullForUnknownId() { + assertNull(interner.get("0".repeat(64))) + } + + @Test + fun getEvictsDeadEntries() { + // Confine the event to a helper so no strong ref leaks into + // this method's stack frame after it returns. The interner's + // WeakReference is then the only thing holding it. + val id = internAndDiscard(interner) + for (i in 0..40) { + System.gc() + Thread.sleep(20) + if (interner.get(id) == null) break + } + assertNull(interner.get(id)) + assertEquals(0, interner.size(), "dead entry should be evicted on access") + } + + private fun internAndDiscard(interner: EventInterner): HexKey { + val ev = signer.sign(TextNoteEvent.build("hello", createdAt = 100)) + interner.intern(ev) + return ev.id + } + + @Test + fun draftChurnDoesNotLeak() { + // 100 distinct drafts, no strong references retained. + val ids = ArrayList(100) + for (i in 0 until 100) { + val e = signer.sign(TextNoteEvent.build("draft-$i", createdAt = 1_000L + i)) + ids.add(e.id) + interner.intern(e) + } + // Force GC + sweep — calling get() on every id triggers + // the on-access cleanup path. + for (i in 0..40) { + System.gc() + Thread.sleep(20) + ids.forEach { interner.get(it) } + if (interner.size() == 0) break + } + assertTrue(interner.size() <= 5, "expected most drafts to be evicted, got ${interner.size()}") + } + + @Test + fun defaultInstanceIsShared() { + val a = signer.sign(TextNoteEvent.build("hello", createdAt = 100)) + val canonical = EventInterner.Default.intern(a) + // A second intern call from anywhere returns the same canonical. + val b = EventInterner.Default.intern(a) + assertSame(canonical, b) + EventInterner.Default.clear() + } +} diff --git a/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.linux.kt b/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.linux.kt new file mode 100644 index 000000000..38df76c6e --- /dev/null +++ b/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.linux.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.core + +/** Linux native actual: passthrough. See `EventInterner.apple.kt`. */ +actual class EventInterner { + actual fun intern(event: Event): Event = event + + actual fun get(id: HexKey): Event? = null + + actual fun size(): Int = 0 + + actual fun clear() {} + + actual companion object { + actual val Default: EventInterner = EventInterner() + } +} From 7f613d600371267b31e15e1c5ffa3286a083b32b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 13:45:01 +0000 Subject: [PATCH 10/24] refactor(quartz): intern only on store reads, inject per-store interner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit interned at Event.fromJson, which is too early — the deserializer can produce events with manipulated id fields that don't match their content. Move interning to the post-validation boundary: events are only canonicalised when read back from durable storage, where they were valid when written. - Revert Event.fromJson to plain OptimizedJsonMapper.fromJson. - Revert ObservableEventStore.insert interning. Substituting the caller's event for a previously-cached one with the same id but potentially different sig was a write-side surprise; we leave the caller's instance alone. - Intern at SQLiteStatement.toEvent (read deserialization). - Intern at FsEventStore.readEvent (FS reads). The interner is also now constructor-injected through every store layer (SQLiteEventStore, EventStore, QueryBuilder, FsEventStore) defaulting to EventInterner.Default. BaseDBTest gives each parallel forEachDB store its own EventInterner — without this, parallel test runs that sign "same id, different sig" events through the shared Default would see the first run's sig leak across all DBs. All 240 store + projection + interner tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../quartz/nip01Core/core/Event.kt | 2 +- .../nip01Core/store/sqlite/EventStore.kt | 4 ++- .../nip01Core/store/sqlite/QueryBuilder.kt | 29 ++++++++++++------- .../store/sqlite/SQLiteEventStore.kt | 3 ++ .../nip01Core/store/sqlite/BaseDBTest.kt | 7 +++++ .../quartz/nip01Core/store/fs/FsEventStore.kt | 7 ++++- 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/Event.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/Event.kt index 576fe6813..906773852 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/Event.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/Event.kt @@ -67,7 +67,7 @@ open class Event( fun toJson(): String = OptimizedJsonMapper.toJson(this) companion object { - fun fromJson(json: String): Event = EventInterner.Default.intern(OptimizedJsonMapper.fromJson(json)) + fun fromJson(json: String): Event = OptimizedJsonMapper.fromJson(json) fun fromJsonOrNull(json: String) = try { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index 0a0f1a75b..ddabf901a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.driver.bundled.BundledSQLiteDriver import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl @@ -36,8 +37,9 @@ class EventStore( dbName: String? = "events.db", val relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(), val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), + val interner: EventInterner = EventInterner.Default, ) : IEventStore { - val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy) + val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy, interner = interner) override suspend fun insert(event: Event) = store.insertEvent(event) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index fa6ad1ffe..51db325bc 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.SQLiteConnection import androidx.sqlite.SQLiteStatement import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.Kind import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper @@ -35,6 +36,7 @@ class QueryBuilder( val fts: FullTextSearchModule, val hasher: (db: SQLiteConnection) -> TagNameValueHasher, val indexStrategy: IndexingStrategy, + val interner: EventInterner = EventInterner.Default, ) { // ------------ // Main methods @@ -232,16 +234,23 @@ class QueryBuilder( } } - private fun SQLiteStatement.toEvent() = - EventFactory.create( - getText(0), - getText(1), - getLong(2), - getInt(3), - OptimizedJsonMapper.fromJsonToTagArray(getText(4)), - getText(5), - getText(6), - ) + @Suppress("UNCHECKED_CAST") + private fun SQLiteStatement.toEvent(): T { + val event = + EventFactory.create( + getText(0), + getText(1), + getLong(2), + getInt(3), + OptimizedJsonMapper.fromJsonToTagArray(getText(4)), + getText(5), + getText(6), + ) + // Events read from durable storage were valid when written, so + // interning their reconstruction is safe. Multiple projections + // re-querying the same id end up sharing one Event instance. + return interner.intern(event) as T + } private fun SQLiteStatement.toRawEvent() = RawEvent( diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 2da5cf49c..cc6e56f71 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -25,6 +25,7 @@ import androidx.sqlite.SQLiteDriver import androidx.sqlite.SQLiteException import androidx.sqlite.driver.bundled.BundledSQLiteDriver import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.Kind import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper @@ -41,6 +42,7 @@ class SQLiteEventStore( val relay: NormalizedRelayUrl? = null, val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), val numReaders: Int = 4, + val interner: EventInterner = EventInterner.Default, ) { companion object { const val DATABASE_VERSION = 2 @@ -68,6 +70,7 @@ class SQLiteEventStore( fullTextSearchModule, seedModule::hasher, indexStrategy, + interner, ) val modules = diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt index 300ceb4cc..a89aaa9b1 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.quartz.nip01Core.store.sqlite +import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.utils.Secp256k1Instance import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -63,6 +64,12 @@ open class BaseDBTest { EventStore( dbName = null, indexStrategy = indexStrategy, + // Each store gets its own interner so + // parallel forEachDB runs don't share a + // canonical Event for the same id — + // tests sign with random sigs that + // would otherwise cross-pollinate. + interner = EventInterner(), ) } } diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index 8da5d14b5..50a606573 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.nip01Core.store.fs import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.isAddressable import com.vitorpamplona.quartz.nip01Core.core.isEphemeral @@ -75,6 +76,7 @@ open class FsEventStore( * verification re-canonicalises — so format is purely a UX choice. */ private val eventToJson: (Event) -> String = Event::toJson, + private val interner: EventInterner = EventInterner.Default, ) : IEventStore { private val layout = FsLayout(root) private val hasher: TagNameValueHasher @@ -508,7 +510,10 @@ open class FsEventStore( val p = layout.canonical(id) if (!p.exists()) return null return try { - Event.fromJson(p.readText()) + // Events on disk were valid when written, so interning + // their reconstruction is safe and lets multiple readers + // of the same id share one Event instance. + interner.intern(Event.fromJson(p.readText())) } catch (_: java.nio.file.NoSuchFileException) { null } From 03e9c9d2b2c87237bd28408e4800e514a05dd745 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 14:19:22 +0000 Subject: [PATCH 11/24] refactor(quartz): move EventInterner into nip01Core/cache package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits the cache primitives off into their own package. Files moved: nip01Core/core/EventInterner.kt → nip01Core/cache/EventInterner.kt nip01Core/core/EventInterner.jvmAndroid → nip01Core/cache/EventInterner.jvmAndroid nip01Core/core/EventInterner.apple → nip01Core/cache/EventInterner.apple nip01Core/core/EventInterner.linux → nip01Core/cache/EventInterner.linux nip01Core/core/EventInternerTest.kt → nip01Core/cache/EventInternerTest.kt Imports updated in SQLiteEventStore, QueryBuilder, EventStore (sqlite), FsEventStore, BaseDBTest. Each platform actual now explicitly imports nip01Core.core.Event / HexKey since we crossed package boundaries. 7/7 interner + 240/240 store + projection tests still pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../quartz/nip01Core/{core => cache}/EventInterner.apple.kt | 5 ++++- .../quartz/nip01Core/{core => cache}/EventInterner.kt | 5 ++++- .../quartz/nip01Core/store/sqlite/EventStore.kt | 2 +- .../quartz/nip01Core/store/sqlite/QueryBuilder.kt | 2 +- .../quartz/nip01Core/store/sqlite/SQLiteEventStore.kt | 2 +- .../quartz/nip01Core/store/sqlite/BaseDBTest.kt | 2 +- .../nip01Core/{core => cache}/EventInterner.jvmAndroid.kt | 4 +++- .../vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt | 2 +- .../quartz/nip01Core/{core => cache}/EventInternerTest.kt | 4 +++- .../quartz/nip01Core/{core => cache}/EventInterner.linux.kt | 5 ++++- 10 files changed, 23 insertions(+), 10 deletions(-) rename quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/{core => cache}/EventInterner.apple.kt (91%) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/{core => cache}/EventInterner.kt (94%) rename quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/{core => cache}/EventInterner.jvmAndroid.kt (95%) rename quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/{core => cache}/EventInternerTest.kt (96%) rename quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/{core => cache}/EventInterner.linux.kt (90%) diff --git a/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.apple.kt b/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.apple.kt similarity index 91% rename from quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.apple.kt rename to quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.apple.kt index 4dc306f84..ba6dc8c40 100644 --- a/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.apple.kt +++ b/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.apple.kt @@ -18,7 +18,10 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.core +package com.vitorpamplona.quartz.nip01Core.cache + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey /** * Apple actual: passthrough. Kotlin/Native has weak refs but no diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.kt similarity index 94% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.kt index c0be04c21..1bb309ea9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.kt @@ -18,7 +18,10 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.core +package com.vitorpamplona.quartz.nip01Core.cache + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey /** * Process-wide interner that canonicalises [Event] instances by id so diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index ddabf901a..ebe8e8641 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -21,8 +21,8 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.driver.bundled.BundledSQLiteDriver +import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 51db325bc..52151be43 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -22,8 +22,8 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.SQLiteConnection import androidx.sqlite.SQLiteStatement +import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.Kind import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index cc6e56f71..107d5cb9d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -24,8 +24,8 @@ import androidx.sqlite.SQLiteConnection import androidx.sqlite.SQLiteDriver import androidx.sqlite.SQLiteException import androidx.sqlite.driver.bundled.BundledSQLiteDriver +import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.Kind import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt index a89aaa9b1..74e7291b6 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt @@ -20,7 +20,7 @@ */ package com.vitorpamplona.quartz.nip01Core.store.sqlite -import com.vitorpamplona.quartz.nip01Core.core.EventInterner +import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.utils.Secp256k1Instance import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.jvmAndroid.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.jvmAndroid.kt similarity index 95% rename from quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.jvmAndroid.kt rename to quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.jvmAndroid.kt index f728608ac..447ff393f 100644 --- a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.jvmAndroid.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.jvmAndroid.kt @@ -18,8 +18,10 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.core +package com.vitorpamplona.quartz.nip01Core.cache +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey import java.lang.ref.WeakReference import java.util.concurrent.ConcurrentHashMap diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index 50a606573..a1ff99573 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -20,9 +20,9 @@ */ package com.vitorpamplona.quartz.nip01Core.store.fs +import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.EventInterner import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.isAddressable import com.vitorpamplona.quartz.nip01Core.core.isEphemeral diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInternerTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInternerTest.kt similarity index 96% rename from quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInternerTest.kt rename to quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInternerTest.kt index f55f79cd8..c8eb078d1 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInternerTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInternerTest.kt @@ -18,8 +18,10 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.core +package com.vitorpamplona.quartz.nip01Core.cache +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.utils.Secp256k1Instance diff --git a/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.linux.kt b/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.linux.kt similarity index 90% rename from quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.linux.kt rename to quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.linux.kt index 38df76c6e..c7f042f37 100644 --- a/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/EventInterner.linux.kt +++ b/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.linux.kt @@ -18,7 +18,10 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.core +package com.vitorpamplona.quartz.nip01Core.cache + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey /** Linux native actual: passthrough. See `EventInterner.apple.kt`. */ actual class EventInterner { From 3fc4781d1296f4408ad66c2c0a4a59b53eab02a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 14:30:48 +0000 Subject: [PATCH 12/24] refactor(quartz): encapsulate interning as InternedEventStore decorator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the ctor-injected interner inside SQLiteEventStore / QueryBuilder / EventStore / FsEventStore with a standalone InternedEventStore decorator in nip01Core/cache/. Composition is now explicit at the call site: val sqlite = EventStore(...) val cached = InternedEventStore(sqlite) val observable = ObservableEventStore(cached) Each layer has one job: EventStore persists, InternedEventStore canonicalises read results, ObservableEventStore publishes the bus. Stores no longer carry an interner field; passing one through three layers of constructor params is gone. InternedEventStore wraps every IEventStore.query variant (list + streaming, single filter + multi) and pipes results through interner.intern. Writes (insert, transaction, delete*, deleteExpiredEvents) and counts pass through untouched. assertQuery test helpers generalized from EventStore / SQLiteEventStore to IEventStore so the decorator works with the existing fixtures. BaseDBTest reverts to plain EventStore — the basic store tests don't read through the projection / interning layer, so they don't need decoration. Tests that DO want canonical-instance reads can wrap explicitly: `InternedEventStore(eventStore)`. 7/7 interner + 240/240 store + projection tests still pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../nip01Core/cache/InternedEventStore.kt | 87 +++++++++++++++++++ .../nip01Core/store/sqlite/EventStore.kt | 4 +- .../nip01Core/store/sqlite/QueryBuilder.kt | 29 +++---- .../store/sqlite/SQLiteEventStore.kt | 3 - .../nip01Core/store/sqlite/AssertUtils.kt | 5 +- .../nip01Core/store/sqlite/BaseDBTest.kt | 7 -- .../quartz/nip01Core/store/fs/FsEventStore.kt | 7 +- 7 files changed, 102 insertions(+), 40 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/InternedEventStore.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/InternedEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/InternedEventStore.kt new file mode 100644 index 000000000..db078e1e5 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/InternedEventStore.kt @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.cache + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.IEventStore + +/** + * Decorator that canonicalises every [Event] returned by the inner + * store through an [EventInterner], so all consumers of read paths + * share a single object reference per event id. + * + * Wrap any [IEventStore] (SQLite, FS, in-memory test fake) when you + * want event-identity sharing across the read paths: + * + * ``` + * val sqlite = EventStore(...) + * val cached = InternedEventStore(sqlite) + * val observable = ObservableEventStore(cached) + * ``` + * + * Writes (`insert`, `transaction`, `delete*`) pass through unchanged + * — the decorator never substitutes the caller's event for a cached + * one on the way in (sigs may differ across "same id, different + * decode" cases). Canonicalisation only happens on results coming + * back out of the store. + * + * The default [interner] is [EventInterner.Default]; pass a fresh + * instance for tests or any context that needs isolation. + */ +class InternedEventStore( + private val inner: IEventStore, + private val interner: EventInterner = EventInterner.Default, +) : IEventStore { + override suspend fun insert(event: Event) = inner.insert(event) + + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = inner.transaction(body) + + @Suppress("UNCHECKED_CAST") + override suspend fun query(filter: Filter): List = inner.query(filter).map { interner.intern(it) as T } + + @Suppress("UNCHECKED_CAST") + override suspend fun query(filters: List): List = inner.query(filters).map { interner.intern(it) as T } + + @Suppress("UNCHECKED_CAST") + override suspend fun query( + filter: Filter, + onEach: (T) -> Unit, + ) = inner.query(filter) { onEach(interner.intern(it) as T) } + + @Suppress("UNCHECKED_CAST") + override suspend fun query( + filters: List, + onEach: (T) -> Unit, + ) = inner.query(filters) { onEach(interner.intern(it) as T) } + + override suspend fun count(filter: Filter): Int = inner.count(filter) + + override suspend fun count(filters: List): Int = inner.count(filters) + + override suspend fun delete(filter: Filter) = inner.delete(filter) + + override suspend fun delete(filters: List) = inner.delete(filters) + + override suspend fun deleteExpiredEvents() = inner.deleteExpiredEvents() + + override fun close() = inner.close() +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index ebe8e8641..0a0f1a75b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -21,7 +21,6 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.driver.bundled.BundledSQLiteDriver -import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -37,9 +36,8 @@ class EventStore( dbName: String? = "events.db", val relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(), val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), - val interner: EventInterner = EventInterner.Default, ) : IEventStore { - val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy, interner = interner) + val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy) override suspend fun insert(event: Event) = store.insertEvent(event) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index 52151be43..6a3412e52 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -22,7 +22,6 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.SQLiteConnection import androidx.sqlite.SQLiteStatement -import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.Kind @@ -36,7 +35,6 @@ class QueryBuilder( val fts: FullTextSearchModule, val hasher: (db: SQLiteConnection) -> TagNameValueHasher, val indexStrategy: IndexingStrategy, - val interner: EventInterner = EventInterner.Default, ) { // ------------ // Main methods @@ -234,23 +232,16 @@ class QueryBuilder( } } - @Suppress("UNCHECKED_CAST") - private fun SQLiteStatement.toEvent(): T { - val event = - EventFactory.create( - getText(0), - getText(1), - getLong(2), - getInt(3), - OptimizedJsonMapper.fromJsonToTagArray(getText(4)), - getText(5), - getText(6), - ) - // Events read from durable storage were valid when written, so - // interning their reconstruction is safe. Multiple projections - // re-querying the same id end up sharing one Event instance. - return interner.intern(event) as T - } + private fun SQLiteStatement.toEvent(): T = + EventFactory.create( + getText(0), + getText(1), + getLong(2), + getInt(3), + OptimizedJsonMapper.fromJsonToTagArray(getText(4)), + getText(5), + getText(6), + ) private fun SQLiteStatement.toRawEvent() = RawEvent( diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 107d5cb9d..2da5cf49c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -24,7 +24,6 @@ import androidx.sqlite.SQLiteConnection import androidx.sqlite.SQLiteDriver import androidx.sqlite.SQLiteException import androidx.sqlite.driver.bundled.BundledSQLiteDriver -import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.Kind @@ -42,7 +41,6 @@ class SQLiteEventStore( val relay: NormalizedRelayUrl? = null, val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), val numReaders: Int = 4, - val interner: EventInterner = EventInterner.Default, ) { companion object { const val DATABASE_VERSION = 2 @@ -70,7 +68,6 @@ class SQLiteEventStore( fullTextSearchModule, seedModule::hasher, indexStrategy, - interner, ) val modules = diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AssertUtils.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AssertUtils.kt index d7fa94ce5..57215924c 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AssertUtils.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AssertUtils.kt @@ -22,9 +22,10 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.IEventStore import kotlin.test.assertEquals -suspend fun EventStore.assertQuery( +suspend fun IEventStore.assertQuery( expected: T?, filter: Filter, ) { @@ -40,7 +41,7 @@ suspend fun EventStore.assertQuery( } } -suspend fun EventStore.assertQuery( +suspend fun IEventStore.assertQuery( expected: List, filter: Filter, ) { diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt index 74e7291b6..300ceb4cc 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.quartz.nip01Core.store.sqlite -import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.utils.Secp256k1Instance import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -64,12 +63,6 @@ open class BaseDBTest { EventStore( dbName = null, indexStrategy = indexStrategy, - // Each store gets its own interner so - // parallel forEachDB runs don't share a - // canonical Event for the same id — - // tests sign with random sigs that - // would otherwise cross-pollinate. - interner = EventInterner(), ) } } diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index a1ff99573..8da5d14b5 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.quartz.nip01Core.store.fs -import com.vitorpamplona.quartz.nip01Core.cache.EventInterner import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey @@ -76,7 +75,6 @@ open class FsEventStore( * verification re-canonicalises — so format is purely a UX choice. */ private val eventToJson: (Event) -> String = Event::toJson, - private val interner: EventInterner = EventInterner.Default, ) : IEventStore { private val layout = FsLayout(root) private val hasher: TagNameValueHasher @@ -510,10 +508,7 @@ open class FsEventStore( val p = layout.canonical(id) if (!p.exists()) return null return try { - // Events on disk were valid when written, so interning - // their reconstruction is safe and lets multiple readers - // of the same id share one Event instance. - interner.intern(Event.fromJson(p.readText())) + Event.fromJson(p.readText()) } catch (_: java.nio.file.NoSuchFileException) { null } From 3792c98ec5787544bfbb33c724cd59dbd23cd099 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 14:38:23 +0000 Subject: [PATCH 13/24] =?UTF-8?q?refactor(quartz):=20rename=20StoreEvent?= =?UTF-8?q?=20=E2=86=92=20StoreChange,=20events=20flow=20=E2=86=92=20chang?= =?UTF-8?q?es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Event" name was overloaded — the store-mutation type lived right next to Nostr Event, so StoreEvent.Insert(event: Event) read awkwardly. Renaming to StoreChange disambiguates and matches reactive vocabulary ("store changes"). - StoreEvent → StoreChange (file, sealed type, all references). - ObservableEventStore.events → ObservableEventStore.changes. - Internal field _events → _changes. - KDoc cross-references updated. Case names (Insert, DeleteByFilter, DeleteExpired) unchanged. All 240 store + projection + interner tests still pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../store/projection/EventStoreProjection.kt | 20 ++++++------- .../store/projection/ObservableEventStore.kt | 28 +++++++++---------- .../{StoreEvent.kt => StoreChange.kt} | 10 +++---- .../projection/EventStoreProjectionTest.kt | 4 +-- 4 files changed, 31 insertions(+), 31 deletions(-) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/{StoreEvent.kt => StoreChange.kt} (93%) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt index 0b20f919d..f70e93ac3 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt @@ -63,10 +63,10 @@ import kotlinx.coroutines.yield * * The seed is materialised by querying the store once at start, after * which the projection is driven entirely by - * [ObservableEventStore.events]. Three kinds of mutation arrive on + * [ObservableEventStore.changes]. Three kinds of mutation arrive on * that stream: * - * - [StoreEvent.Insert] — interpreted in-projection so a single + * - [StoreChange.Insert] — interpreted in-projection so a single * arriving event can carry NIP-01 / NIP-09 / NIP-62 semantics: * - **NIP-01 supersession.** New replaceable / addressable events * replace prior ones for the same `kind:pubkey[:dtag]`. The @@ -82,11 +82,11 @@ import kotlinx.coroutines.yield * already lapsed at the moment they arrive are dropped before * they ever enter [items]. * - * - [StoreEvent.DeleteByFilter] — emitted on `delete(filter)` / + * - [StoreChange.DeleteByFilter] — emitted on `delete(filter)` / * `delete(filters)`. The projection drops every slot matching any * of the rule's filters via [Filter.match]. * - * - [StoreEvent.DeleteExpired] — emitted on `deleteExpiredEvents()`. + * - [StoreChange.DeleteExpired] — emitted on `deleteExpiredEvents()`. * The projection drops every slot whose `expiration` has lapsed at * the cutoff the store pinned. **There is no per-projection * expiration ticker** — projections only drop expired events when @@ -101,7 +101,7 @@ import kotlinx.coroutines.yield * filter under cap, it stays under cap until another match arrives. * * Ephemeral events (kinds `20000-29999`) reach the projection via - * [ObservableEventStore.events] without ever being persisted; they + * [ObservableEventStore.changes] without ever being persisted; they * appear in [items] for as long as the projection is alive but never * survive a re-seed. They aren't covered by the store's * `deleteExpiredEvents()` sweep (the DB never had them), so an @@ -147,7 +147,7 @@ class EventStoreProjection( // emissions arriving during seed and drains them once collect // proceeds. Doing the seed inside `collect { }` would race // with concurrent inserts. - store.events + store.changes .onSubscription { seed() ready.complete(Unit) @@ -167,20 +167,20 @@ class EventStoreProjection( publish() } - private fun apply(storeEvent: StoreEvent) { + private fun apply(storeEvent: StoreChange) { val changed = when (storeEvent) { - is StoreEvent.Insert -> { + is StoreChange.Insert -> { applyInsert(storeEvent.event) } - is StoreEvent.DeleteByFilter -> { + is StoreChange.DeleteByFilter -> { dropWhere { ev -> storeEvent.filters.any { it.match(ev) } } } // Store's sweep uses strict `<`; isExpirationBefore is // `<=`, so subtract 1 to match. - is StoreEvent.DeleteExpired -> { + is StoreChange.DeleteExpired -> { val cutoff = (storeEvent.asOf ?: nowProvider()) - 1 dropWhere { it.isExpirationBefore(cutoff) } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt index d9c4699f6..0355d9710 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt @@ -44,26 +44,26 @@ import kotlinx.coroutines.flow.asSharedFlow * - **Non-ephemeral events** are forwarded to the inner store. If the * inner store rejects (expired, NIP-09 / NIP-62 tombstone, NIP-01 * supersession loser), the rejection propagates and nothing is - * emitted on [events]. + * emitted on [changes]. * - **Ephemeral events** (kinds `20000-29999`) skip the inner store * entirely — they're never persisted — but they still emit on - * [events] so projections can render them while they live. Already + * [changes] so projections can render them while they live. Already * expired ephemerals are silently dropped. * * Wrap any store you want to observe — [SQLiteEventStore], FS-backed, * an in-memory test fake — and feed [EventStoreProjection] from the - * resulting [events] flow. + * resulting [changes] flow. * * Reads (`query`, `count`) and out-of-band writes (`delete`, * `deleteExpiredEvents`) forward to the inner store unchanged. The - * latter are *not* surfaced on [events] — see the projection's + * latter are *not* surfaced on [changes] — see the projection's * docstring for the rationale. */ class ObservableEventStore( val inner: IEventStore, ) : IEventStore { - private val _events = - MutableSharedFlow( + private val _changes = + MutableSharedFlow( replay = 0, extraBufferCapacity = 256, onBufferOverflow = BufferOverflow.SUSPEND, @@ -77,9 +77,9 @@ class ObservableEventStore( * transactions emit nothing. * * Projections consume this stream — see [EventStoreProjection] - * for how each [StoreEvent] is interpreted. + * for how each [StoreChange] is interpreted. */ - val events: SharedFlow = _events.asSharedFlow() + val changes: SharedFlow = _changes.asSharedFlow() override suspend fun insert(event: Event) { if (event.kind.isEphemeral()) { @@ -87,13 +87,13 @@ class ObservableEventStore( // that are already expired — they were never going to live // long enough for a UI to render them. if (event.isExpired()) return - _events.emit(StoreEvent.Insert(event)) + _changes.emit(StoreChange.Insert(event)) return } // Non-ephemeral: let the inner store enforce expiration, // tombstones, supersession, etc. If it throws, we never emit. inner.insert(event) - _events.emit(StoreEvent.Insert(event)) + _changes.emit(StoreChange.Insert(event)) } override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { @@ -120,7 +120,7 @@ class ObservableEventStore( } // Emit only after the inner transaction commits. If it throws // / rolls back, `accepted` is discarded. - for (e in accepted) _events.emit(StoreEvent.Insert(e)) + for (e in accepted) _changes.emit(StoreChange.Insert(e)) } override suspend fun query(filter: Filter): List = inner.query(filter) @@ -143,12 +143,12 @@ class ObservableEventStore( override suspend fun delete(filter: Filter) { inner.delete(filter) - _events.emit(StoreEvent.DeleteByFilter(listOf(filter))) + _changes.emit(StoreChange.DeleteByFilter(listOf(filter))) } override suspend fun delete(filters: List) { inner.delete(filters) - _events.emit(StoreEvent.DeleteByFilter(filters)) + _changes.emit(StoreChange.DeleteByFilter(filters)) } override suspend fun deleteExpiredEvents() { @@ -159,7 +159,7 @@ class ObservableEventStore( // own clock when the event is processed. val asOf = TimeUtils.now() inner.deleteExpiredEvents() - _events.emit(StoreEvent.DeleteExpired(asOf)) + _changes.emit(StoreChange.DeleteExpired(asOf)) } /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreChange.kt similarity index 93% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreEvent.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreChange.kt index e0b66ba46..c268583f8 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreEvent.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreChange.kt @@ -24,7 +24,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter /** - * Mutations published by [ObservableEventStore.events]. Projections + * Mutations published by [ObservableEventStore.changes]. Projections * react to these to keep their in-memory view in sync with the * underlying store. * @@ -42,16 +42,16 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter * pin the timestamp it actually used, so the projection drops * exactly the events the store dropped. */ -sealed interface StoreEvent { +sealed interface StoreChange { data class Insert( val event: Event, - ) : StoreEvent + ) : StoreChange data class DeleteByFilter( val filters: List, - ) : StoreEvent + ) : StoreChange data class DeleteExpired( val asOf: Long? = null, - ) : StoreEvent + ) : StoreChange } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt index a5811f220..259e053f6 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt @@ -373,7 +373,7 @@ class EventStoreProjectionTest { // Let the short expiration lapse, then ask the store to // sweep — the projection drops the expired slot in - // response to the resulting StoreEvent.Delete(Expired). + // response to the resulting StoreChange.Delete(Expired). delay(2000) observable.deleteExpiredEvents() @@ -523,7 +523,7 @@ class EventStoreProjectionTest { * Ephemeral events (kinds `20000-29999`) are never persisted, so * the inner SQLite store silently drops them. The * `ObservableEventStore` wrapper still routes them onto its - * [events][com.vitorpamplona.quartz.nip01Core.store.projection.ObservableEventStore.events] + * [events][com.vitorpamplona.quartz.nip01Core.store.projection.ObservableEventStore.changes] * flow, so an open projection sees them while it's alive. They * vanish from any future seed because the DB never had them. */ From e126ca679a5175a70bb811706d9bed050000a1c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 14:40:15 +0000 Subject: [PATCH 14/24] refactor(quartz): move interning classes into nip01Core/cache/interning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groups EventInterner + InternedEventStore + the platform actuals + the test under a focused interning sub-package. Leaves nip01Core/cache/ as the namespace for cache primitives generally; future cache types (LargeWeakCache, etc.) get sibling sub-packages. Files moved: cache/EventInterner.kt → cache/interning/EventInterner.kt cache/InternedEventStore.kt → cache/interning/InternedEventStore.kt cache/EventInterner.jvmAndroid → cache/interning/EventInterner.jvmAndroid cache/EventInterner.apple → cache/interning/EventInterner.apple cache/EventInterner.linux → cache/interning/EventInterner.linux cache/EventInternerTest.kt → cache/interning/EventInternerTest.kt No consumers to update — the previous decorator refactor already removed the interner from all the store ctors. 7/7 interner + 240/240 store + projection tests still pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../nip01Core/cache/{ => interning}/EventInterner.apple.kt | 2 +- .../quartz/nip01Core/cache/{ => interning}/EventInterner.kt | 2 +- .../nip01Core/cache/{ => interning}/InternedEventStore.kt | 2 +- .../nip01Core/cache/{ => interning}/EventInterner.jvmAndroid.kt | 2 +- .../quartz/nip01Core/cache/{ => interning}/EventInternerTest.kt | 2 +- .../nip01Core/cache/{ => interning}/EventInterner.linux.kt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/{ => interning}/EventInterner.apple.kt (96%) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/{ => interning}/EventInterner.kt (97%) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/{ => interning}/InternedEventStore.kt (98%) rename quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/{ => interning}/EventInterner.jvmAndroid.kt (98%) rename quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/{ => interning}/EventInternerTest.kt (98%) rename quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/{ => interning}/EventInterner.linux.kt (96%) diff --git a/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.apple.kt b/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.apple.kt similarity index 96% rename from quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.apple.kt rename to quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.apple.kt index ba6dc8c40..a28c22be6 100644 --- a/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.apple.kt +++ b/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.apple.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.cache +package com.vitorpamplona.quartz.nip01Core.cache.interning import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.kt similarity index 97% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.kt index 1bb309ea9..fe0d02005 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.cache +package com.vitorpamplona.quartz.nip01Core.cache.interning import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/InternedEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt similarity index 98% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/InternedEventStore.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt index db078e1e5..8403bf8ef 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/InternedEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.cache +package com.vitorpamplona.quartz.nip01Core.cache.interning import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.jvmAndroid.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.jvmAndroid.kt similarity index 98% rename from quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.jvmAndroid.kt rename to quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.jvmAndroid.kt index 447ff393f..d393dd2c1 100644 --- a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.jvmAndroid.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.jvmAndroid.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.cache +package com.vitorpamplona.quartz.nip01Core.cache.interning import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInternerTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInternerTest.kt similarity index 98% rename from quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInternerTest.kt rename to quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInternerTest.kt index c8eb078d1..efde104d8 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInternerTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInternerTest.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.cache +package com.vitorpamplona.quartz.nip01Core.cache.interning import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper diff --git a/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.linux.kt b/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.linux.kt similarity index 96% rename from quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.linux.kt rename to quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.linux.kt index c7f042f37..8c6e6223b 100644 --- a/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/EventInterner.linux.kt +++ b/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.linux.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.cache +package com.vitorpamplona.quartz.nip01Core.cache.interning import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey From ecbb7ea10cdc18a7c64dd7ff77e25ab0cc6f97de Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 14:51:16 +0000 Subject: [PATCH 15/24] refactor(quartz): tighten projection package layout + relay flows from store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes wrapped together: 1. relay moves up to IEventStore. The interface gains an abstract `val relay: NormalizedRelayUrl?` and SQLiteEventStore, EventStore, FsEventStore, InternedEventStore, and ObservableEventStore all override it (decorators forward inner.relay). EventStoreProjection loses its `relay` ctor param and uses `store.relay` for NIP-62 vanish scoping; ObservableEventStore.observe overloads lose their relay arg. The relay was always a property of the store anyway — threading it through projection construction was redundant. 2. StoreChange inlines into ObservableEventStore. The sealed type is now ObservableEventStore.StoreChange (with Insert / DeleteByFilter / DeleteExpired as nested cases) since it's strictly a contract of the change stream, never used independently. StoreChange.kt deleted. 3. Package reshuffle: store/projection/ObservableEventStore.kt → store/ObservableEventStore.kt store/projection/EventStoreProjection.kt → cache/projection/EventStoreProjection.kt store/projection/StoreChange.kt → inlined into ObservableEventStore ObservableEventStore is a store decorator and lives next to IEventStore. EventStoreProjection is a cache primitive (alongside EventInterner / InternedEventStore) and lives under cache/. The `observe(filters, scope)` convenience moved from a method on ObservableEventStore to a top-level extension function in cache/projection/, which avoids the otherwise-circular store → cache/projection → store dependency. 7/7 interner + 17/17 projection + all other store tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../cache/interning/InternedEventStore.kt | 3 + .../projection/EventStoreProjection.kt | 28 ++++++-- .../quartz/nip01Core/store/IEventStore.kt | 9 +++ .../{projection => }/ObservableEventStore.kt | 72 +++++++++++-------- .../nip01Core/store/projection/StoreChange.kt | 57 --------------- .../nip01Core/store/sqlite/EventStore.kt | 2 +- .../projection/EventStoreProjectionTest.kt | 32 ++++----- .../quartz/nip01Core/store/fs/FsEventStore.kt | 2 +- 8 files changed, 91 insertions(+), 114 deletions(-) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/{store => cache}/projection/EventStoreProjection.kt (94%) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/{projection => }/ObservableEventStore.kt (77%) delete mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreChange.kt rename quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/{store => cache}/projection/EventStoreProjectionTest.kt (95%) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt index 8403bf8ef..2bea9daae 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.nip01Core.cache.interning import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore /** @@ -51,6 +52,8 @@ class InternedEventStore( private val inner: IEventStore, private val interner: EventInterner = EventInterner.Default, ) : IEventStore { + override val relay: NormalizedRelayUrl? get() = inner.relay + override suspend fun insert(event: Event) = inner.insert(event) override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = inner.transaction(body) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt similarity index 94% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt index f70e93ac3..36ec6b9d1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.store.projection +package com.vitorpamplona.quartz.nip01Core.cache.projection import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent @@ -26,7 +26,8 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.isReplaceable import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore +import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore.StoreChange import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip40Expiration.isExpirationBefore import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent @@ -76,8 +77,8 @@ import kotlinx.coroutines.yield * handle owned by the same author (for GiftWrap, the recipient). * Cross-author deletions are inert. * - **NIP-62 right to vanish.** A [RequestToVanishEvent] whose - * `shouldVanishFrom([relay])` is true drops every handle from - * the same author with `created_at < vanish.created_at`. + * `shouldVanishFrom(store.relay)` is true drops every handle + * from the same author with `created_at < vanish.created_at`. * - **NIP-40 expiration.** Events whose `expiration` tag has * already lapsed at the moment they arrive are dropped before * they ever enter [items]. @@ -115,7 +116,6 @@ import kotlinx.coroutines.yield class EventStoreProjection( private val store: ObservableEventStore, private val filters: List, - private val relay: NormalizedRelayUrl?, scope: CoroutineScope, private val nowProvider: () -> Long = TimeUtils::now, ) : AutoCloseable { @@ -199,7 +199,7 @@ class EventStoreProjection( if (event is DeletionEvent) { if (handleDeletion(event)) changed = true } - if (event is RequestToVanishEvent && event.shouldVanishFrom(relay)) { + if (event is RequestToVanishEvent && event.shouldVanishFrom(store.relay)) { if (dropWhere { ev -> ownerOf(ev) == event.pubKey && ev.createdAt < event.createdAt }) changed = true } @@ -408,3 +408,19 @@ class EventStoreProjection( } } } + +/** + * Convenience: open a projection over this observable store. NIP-62 + * vanish handling is scoped by the inner store's `relay`. Cancel + * [scope] (or call [EventStoreProjection.close]) to release the + * projection. + */ +fun ObservableEventStore.observe( + filters: List, + scope: CoroutineScope, +): EventStoreProjection = EventStoreProjection(this, filters, scope) + +fun ObservableEventStore.observe( + filter: Filter, + scope: CoroutineScope, +): EventStoreProjection = EventStoreProjection(this, listOf(filter), scope) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index e70773c29..d376b8e7d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -22,8 +22,17 @@ package com.vitorpamplona.quartz.nip01Core.store import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl interface IEventStore : AutoCloseable { + /** + * Relay URL this store is acting on behalf of, or `null` for an + * unscoped store. Used by NIP-62 right-to-vanish handling: only + * vanish requests whose `relays` list contains this URL (or + * `ALL_RELAYS`) cascade. + */ + val relay: NormalizedRelayUrl? + suspend fun insert(event: Event) interface ITransaction { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt similarity index 77% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt index 0355d9710..a65c1e5e1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt @@ -18,16 +18,14 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.store.projection +package com.vitorpamplona.quartz.nip01Core.store import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.isEphemeral import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip40Expiration.isExpired import com.vitorpamplona.quartz.utils.TimeUtils -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.SharedFlow @@ -50,18 +48,21 @@ import kotlinx.coroutines.flow.asSharedFlow * [changes] so projections can render them while they live. Already * expired ephemerals are silently dropped. * - * Wrap any store you want to observe — [SQLiteEventStore], FS-backed, - * an in-memory test fake — and feed [EventStoreProjection] from the + * Wrap any store you want to observe — `SQLiteEventStore`, FS-backed, + * an in-memory test fake — and feed `EventStoreProjection` from the * resulting [changes] flow. * * Reads (`query`, `count`) and out-of-band writes (`delete`, - * `deleteExpiredEvents`) forward to the inner store unchanged. The - * latter are *not* surfaced on [changes] — see the projection's - * docstring for the rationale. + * `deleteExpiredEvents`) forward to the inner store; the latter are + * also surfaced on [changes] as [StoreChange.DeleteByFilter] / + * [StoreChange.DeleteExpired] so projections can drop the matching + * slots in memory without re-querying. */ class ObservableEventStore( val inner: IEventStore, ) : IEventStore { + override val relay: NormalizedRelayUrl? get() = inner.relay + private val _changes = MutableSharedFlow( replay = 0, @@ -72,15 +73,47 @@ class ObservableEventStore( /** * Stream of mutations accepted by the observable layer. One * emission per successful [insert] (or per accepted event in a - * [transaction] body), one emission per [delete] / [delete] / + * [transaction] body), one emission per [delete] / * [deleteExpiredEvents] call. Rejected inserts and rolled-back * transactions emit nothing. * - * Projections consume this stream — see [EventStoreProjection] + * Projections consume this stream — see `EventStoreProjection` * for how each [StoreChange] is interpreted. */ val changes: SharedFlow = _changes.asSharedFlow() + /** + * Mutations published by [changes]. Projections react to these to + * keep their in-memory view in sync with the underlying store. + * + * - [Insert] is emitted for every event accepted by the + * observable layer (persistable or ephemeral). Carries the + * event itself so the projection can run its NIP-01 / NIP-09 / + * NIP-62 interpretation. + * - [DeleteByFilter] is emitted for every `delete(filter)` / + * `delete(filters)` call on the observable. Carries the same + * filters the store used so projections can apply + * [Filter.match] in memory and drop the matching slots without + * re-querying. + * - [DeleteExpired] is emitted for every `deleteExpiredEvents()` + * sweep. The optional [DeleteExpired.asOf] cutoff lets the + * store pin the timestamp it actually used, so the projection + * drops exactly the events the store dropped. + */ + sealed interface StoreChange { + data class Insert( + val event: Event, + ) : StoreChange + + data class DeleteByFilter( + val filters: List, + ) : StoreChange + + data class DeleteExpired( + val asOf: Long? = null, + ) : StoreChange + } + override suspend fun insert(event: Event) { if (event.kind.isEphemeral()) { // Ephemeral kinds bypass persistence. We still drop ones @@ -162,24 +195,5 @@ class ObservableEventStore( _changes.emit(StoreChange.DeleteExpired(asOf)) } - /** - * Open a reactive [EventStoreProjection] over this observable - * store. [relay] scopes NIP-62 vanish handling — pass the relay - * URL the events are arriving from, or `null` to apply only - * unscoped (`ALL_RELAYS`) vanish requests. Cancel [scope] (or - * call [EventStoreProjection.close]) to release the projection. - */ - fun observe( - filters: List, - relay: NormalizedRelayUrl?, - scope: CoroutineScope, - ): EventStoreProjection = EventStoreProjection(this, filters, relay, scope) - - fun observe( - filter: Filter, - relay: NormalizedRelayUrl?, - scope: CoroutineScope, - ): EventStoreProjection = observe(listOf(filter), relay, scope) - override fun close() = inner.close() } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreChange.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreChange.kt deleted file mode 100644 index c268583f8..000000000 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/StoreChange.kt +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store.projection - -import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter - -/** - * Mutations published by [ObservableEventStore.changes]. Projections - * react to these to keep their in-memory view in sync with the - * underlying store. - * - * - [Insert] is emitted for every event accepted by the observable - * layer (persistable or ephemeral). Carries the event itself so - * the projection can run its NIP-01 / NIP-09 / NIP-62 - * interpretation. - * - [DeleteByFilter] is emitted for every `delete(filter)` / - * `delete(filters)` call on the observable. Carries the same - * filters the store used so projections can apply - * [Filter.match] in memory and drop the matching slots without - * re-querying. - * - [DeleteExpired] is emitted for every `deleteExpiredEvents()` - * sweep. The optional [DeleteExpired.asOf] cutoff lets the store - * pin the timestamp it actually used, so the projection drops - * exactly the events the store dropped. - */ -sealed interface StoreChange { - data class Insert( - val event: Event, - ) : StoreChange - - data class DeleteByFilter( - val filters: List, - ) : StoreChange - - data class DeleteExpired( - val asOf: Long? = null, - ) : StoreChange -} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt index 0a0f1a75b..ef9510611 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt @@ -34,7 +34,7 @@ import com.vitorpamplona.quartz.nip01Core.store.IEventStore */ class EventStore( dbName: String? = "events.db", - val relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(), + override val relay: NormalizedRelayUrl? = "wss://quartz.local/".normalizeRelayUrl(), val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), ) : IEventStore { val store = SQLiteEventStore(BundledSQLiteDriver(), dbName, relay, indexStrategy) diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt similarity index 95% rename from quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt rename to quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt index 259e053f6..70edbdef5 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt @@ -18,14 +18,14 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.store.projection +package com.vitorpamplona.quartz.nip01Core.cache.projection import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync -import com.vitorpamplona.quartz.nip01Core.store.projection.ObservableEventStore +import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent @@ -96,7 +96,7 @@ class EventStoreProjectionTest { observable.insert(a) observable.insert(b) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() val items = projection.items.value @@ -112,7 +112,7 @@ class EventStoreProjectionTest { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) observable.insert(a) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() val before = projection.items.value assertEquals(1, before.size) @@ -132,7 +132,7 @@ class EventStoreProjectionTest { val text = signer.sign(TextNoteEvent.build("a", createdAt = 100)) observable.insert(text) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() val seed = projection.items.value @@ -154,7 +154,6 @@ class EventStoreProjectionTest { val projection = observable.observe( Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), - store.relay, scope, ) projection.ready.await() @@ -186,7 +185,6 @@ class EventStoreProjectionTest { authors = listOf(v1.pubKey), tags = mapOf("d" to listOf("blog")), ), - store.relay, scope, ) projection.ready.await() @@ -222,7 +220,6 @@ class EventStoreProjectionTest { val projection = observable.observe( Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), - store.relay, scope, ) projection.ready.await() @@ -251,7 +248,7 @@ class EventStoreProjectionTest { observable.insert(a) observable.insert(b) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() assertEquals(2, projection.items.value.size) @@ -273,7 +270,7 @@ class EventStoreProjectionTest { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) observable.insert(a) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() val seed = projection.items.value assertEquals(1, seed.size) @@ -301,7 +298,7 @@ class EventStoreProjectionTest { observable.insert(a) observable.insert(b) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() assertEquals(2, projection.items.value.size) @@ -330,7 +327,7 @@ class EventStoreProjectionTest { val a = signer.sign(TextNoteEvent.build("a", createdAt = time)) observable.insert(a) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() val seed = projection.items.value @@ -365,7 +362,6 @@ class EventStoreProjectionTest { val projection = observable.observe( Filter(kinds = listOf(TextNoteEvent.KIND)), - store.relay, scope, ) projection.ready.await() @@ -400,7 +396,6 @@ class EventStoreProjectionTest { val projection = observable.observe( Filter(kinds = listOf(TextNoteEvent.KIND)), - store.relay, scope, ) projection.ready.await() @@ -426,7 +421,6 @@ class EventStoreProjectionTest { val projection = observable.observe( Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), - store.relay, scope, ) projection.ready.await() @@ -466,7 +460,6 @@ class EventStoreProjectionTest { val projection = observable.observe( listOf(filterA, filterB), - store.relay, scope, ) projection.ready.await() @@ -490,7 +483,6 @@ class EventStoreProjectionTest { val projection = observable.observe( Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), - store.relay, scope, ) projection.ready.await() @@ -510,7 +502,7 @@ class EventStoreProjectionTest { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) observable.insert(a) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), store.relay, scope) + val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) projection.ready.await() projection.close() @@ -532,7 +524,7 @@ class EventStoreProjectionTest { runBlocking { val ephemeralKind = 22_000 val projection = - observable.observe(Filter(kinds = listOf(ephemeralKind)), store.relay, scope) + observable.observe(Filter(kinds = listOf(ephemeralKind)), scope) projection.ready.await() assertTrue(projection.items.value.isEmpty()) @@ -555,7 +547,7 @@ class EventStoreProjectionTest { // A fresh projection on the same store gets nothing — the // event was only ever live, not durable. val freshProjection = - observable.observe(Filter(kinds = listOf(ephemeralKind)), store.relay, scope) + observable.observe(Filter(kinds = listOf(ephemeralKind)), scope) freshProjection.ready.await() assertTrue(freshProjection.items.value.isEmpty()) diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt index 8da5d14b5..c91cf7445 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsEventStore.kt @@ -66,7 +66,7 @@ open class FsEventStore( * `null`, only "ALL_RELAYS" vanish requests cascade — matches * `SQLiteEventStore`'s relay arg semantics. */ - private val relay: NormalizedRelayUrl? = null, + override val relay: NormalizedRelayUrl? = null, /** * How to render an event to JSON before writing the canonical file. * Default is the compact NIP-01 form ([Event.toJson]); CLIs that From f23537a49c199af7489b2fa4e63cdfe41d773abb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 15:02:30 +0000 Subject: [PATCH 16/24] =?UTF-8?q?refactor(quartz):=20rename=20InternedEven?= =?UTF-8?q?tStore=20=E2=86=92=20InterningEventStore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Past-participle "Interned" suggested an attribute of the events flowing through; present-participle "Interning" describes what the decorator actively does. Read clearer. File renamed, no consumers needed updating (this commit is the only external reference, since callers compose the decorator inline). All 240 store + projection + interner tests still pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../{InternedEventStore.kt => InterningEventStore.kt} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/{InternedEventStore.kt => InterningEventStore.kt} (98%) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt similarity index 98% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt index 2bea9daae..3e521ee22 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InternedEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt @@ -35,7 +35,7 @@ import com.vitorpamplona.quartz.nip01Core.store.IEventStore * * ``` * val sqlite = EventStore(...) - * val cached = InternedEventStore(sqlite) + * val cached = InterningEventStore(sqlite) * val observable = ObservableEventStore(cached) * ``` * @@ -48,7 +48,7 @@ import com.vitorpamplona.quartz.nip01Core.store.IEventStore * The default [interner] is [EventInterner.Default]; pass a fresh * instance for tests or any context that needs isolation. */ -class InternedEventStore( +class InterningEventStore( private val inner: IEventStore, private val interner: EventInterner = EventInterner.Default, ) : IEventStore { From 8517bae7a4c0a4f562d2fcea52fa2dda060c6f0e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 15:18:56 +0000 Subject: [PATCH 17/24] feat(quartz): replace items + ready with sealed ProjectionState MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EventStoreProjection now exposes `state: StateFlow>` instead of separate `items` + `ready` signals. ProjectionState is a sealed interface with two cases: data object Loading // initial state data class Loaded(items: List>) // post-seed view This lets the UI distinguish "still seeding" from "seeded but empty" — the previous `emptyList()` initial value conflated the two. Future extension to `Failed(throwable)` is a single case addition. The `ready: CompletableDeferred` signal is gone; callers use `state.first { it is Loaded }` (or the test `awaitReady()` helper). Test fixtures gain two private extensions on EventStoreProjection: - val items: List> — terse access to the loaded list (returns empty if still seeding). - suspend awaitReady() — replaces ready.await(). awaitItems(predicate) now matches against the Loaded state. 17/17 projection + 240/240 store + interner tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../cache/projection/EventStoreProjection.kt | 40 ++++++-- .../projection/EventStoreProjectionTest.kt | 98 ++++++++++--------- 2 files changed, 84 insertions(+), 54 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt index 36ec6b9d1..40e07a416 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt @@ -33,7 +33,6 @@ import com.vitorpamplona.quartz.nip40Expiration.isExpirationBefore import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.TimeUtils -import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.flow.MutableStateFlow @@ -43,6 +42,23 @@ import kotlinx.coroutines.flow.onSubscription import kotlinx.coroutines.launch import kotlinx.coroutines.yield +/** + * Lifecycle state of an [EventStoreProjection]'s [state][EventStoreProjection.state]. + * + * - [Loading] is the initial state before the seed query completes. + * The UI should show a spinner / skeleton here. + * - [Loaded] holds the current deduped, sorted list of slots. Every + * membership change publishes a fresh [Loaded] instance; in-place + * addressable updates do *not* publish (the slots' own flows do). + */ +sealed interface ProjectionState { + data object Loading : ProjectionState + + data class Loaded( + val items: List>, + ) : ProjectionState +} + /** * A reactive projection over an [ObservableEventStore] for a fixed * set of [filters]. Each visible event is wrapped in a @@ -119,8 +135,16 @@ class EventStoreProjection( scope: CoroutineScope, private val nowProvider: () -> Long = TimeUtils::now, ) : AutoCloseable { - private val _items = MutableStateFlow>>(emptyList()) - val items: StateFlow>> = _items.asStateFlow() + private val _state = MutableStateFlow>(ProjectionState.Loading) + + /** + * Current projection state. Starts as [ProjectionState.Loading] + * until the seed query completes; thereafter every membership + * change publishes a fresh [ProjectionState.Loaded] with the new + * list. Distinguishing `Loading` from `Loaded(emptyList())` lets + * the UI tell "still seeding" apart from "seeded but no matches". + */ + val state: StateFlow> = _state.asStateFlow() /** Slots keyed by the *current* event id. Re-keyed when a replaceable / addressable handle takes a new version. */ private val byId = HashMap>() @@ -137,9 +161,6 @@ class EventStoreProjection( private val perFilter: Map>> = filters.associateWith { sortedSetOf(slotComparator()) } - /** Set when the seed has been written to [items], so callers can suspend until the projection is hot. */ - val ready: CompletableDeferred = CompletableDeferred() - private val collectorJob: Job = scope.launch { // `onSubscription` runs after the SharedFlow subscription is @@ -150,7 +171,6 @@ class EventStoreProjection( store.changes .onSubscription { seed() - ready.complete(Unit) }.collect { storeEvent -> apply(storeEvent) } } @@ -326,12 +346,12 @@ class EventStoreProjection( // sets. Cheaper than maintaining a separate `ordered` field // alongside every insert / remove. if (byId.isEmpty()) { - _items.value = emptyList() + _state.value = ProjectionState.Loaded(emptyList()) return } val union = sortedSetOf(slotComparator()) for (set in perFilter.values) union.addAll(set) - _items.value = union.map { it.flow } + _state.value = ProjectionState.Loaded(union.map { it.flow }) } /** @@ -344,7 +364,7 @@ class EventStoreProjection( byId.clear() byAddress.clear() for (set in perFilter.values) set.clear() - _items.value = emptyList() + _state.value = ProjectionState.Loaded(emptyList()) } /** diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt index 70edbdef5..4112b908b 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt @@ -71,12 +71,22 @@ class EventStoreProjectionTest { store.close() } + /** Snapshot of the currently-loaded items, or empty if still seeding. */ + private val EventStoreProjection.items: List> + get() = (state.value as? ProjectionState.Loaded)?.items.orEmpty() + + /** Suspends until the seed completes; returns the loaded list. */ + private suspend fun EventStoreProjection.awaitReady(timeoutMs: Long = 5_000): List> = + withTimeout(timeoutMs) { + (state.first { it is ProjectionState.Loaded } as ProjectionState.Loaded).items + } + private suspend fun EventStoreProjection.awaitItems( timeoutMs: Long = 5_000, predicate: (List>) -> Boolean, ): List> = withTimeout(timeoutMs) { - items.first { predicate(it) } + (state.first { it is ProjectionState.Loaded && predicate(it.items) } as ProjectionState.Loaded).items } private suspend fun awaitFlow( @@ -97,9 +107,9 @@ class EventStoreProjectionTest { observable.insert(b) val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() + projection.awaitReady() - val items = projection.items.value + val items = projection.items assertEquals(2, items.size) assertEquals(b.id, items[0].value.id) assertEquals(a.id, items[1].value.id) @@ -113,8 +123,8 @@ class EventStoreProjectionTest { observable.insert(a) val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() - val before = projection.items.value + projection.awaitReady() + val before = projection.items assertEquals(1, before.size) val b = signer.sign(TextNoteEvent.build("b", createdAt = 200)) @@ -133,14 +143,14 @@ class EventStoreProjectionTest { observable.insert(text) val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() - val seed = projection.items.value + projection.awaitReady() + val seed = projection.items val meta = signer.sign(MetadataEvent.createNew("Vitor", createdAt = 200)) observable.insert(meta) delay(150) - assertSame(seed, projection.items.value) + assertSame(seed, projection.items) projection.close() } @@ -156,8 +166,8 @@ class EventStoreProjectionTest { Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), scope, ) - projection.ready.await() - val seedList = projection.items.value + projection.awaitReady() + val seedList = projection.items assertEquals(1, seedList.size) val slot = seedList[0] assertEquals(v1.id, slot.value.id) @@ -166,8 +176,8 @@ class EventStoreProjectionTest { observable.insert(v2) awaitFlow(slot) { it.id == v2.id } - assertSame(seedList, projection.items.value, "replaceable update must not change list reference") - assertSame(slot, projection.items.value[0]) + assertSame(seedList, projection.items, "replaceable update must not change list reference") + assertSame(slot, projection.items[0]) projection.close() } @@ -187,15 +197,15 @@ class EventStoreProjectionTest { ), scope, ) - projection.ready.await() - val seedList = projection.items.value + projection.awaitReady() + val seedList = projection.items val slot = seedList[0] val v2 = signer.sign(LongTextNoteEvent.build("blog v2", "title", dTag = "blog", createdAt = time + 1)) observable.insert(v2) awaitFlow(slot) { it.id == v2.id } - assertSame(seedList, projection.items.value, "addressable update must not change list reference") + assertSame(seedList, projection.items, "addressable update must not change list reference") projection.close() } @@ -222,8 +232,8 @@ class EventStoreProjectionTest { Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), scope, ) - projection.ready.await() - val slot = projection.items.value[0] + projection.awaitReady() + val slot = projection.items[0] assertEquals(v2.id, slot.value.id) // The store rejects v1 because v2 already won; the @@ -249,8 +259,8 @@ class EventStoreProjectionTest { observable.insert(b) val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() - assertEquals(2, projection.items.value.size) + projection.awaitReady() + assertEquals(2, projection.items.size) val deletion = signer.sign(DeletionEvent.build(listOf(a))) observable.insert(deletion) @@ -271,8 +281,8 @@ class EventStoreProjectionTest { observable.insert(a) val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() - val seed = projection.items.value + projection.awaitReady() + val seed = projection.items assertEquals(1, seed.size) val foreignDeletion = otherSigner.sign(DeletionEvent.build(listOf(a))) @@ -280,10 +290,10 @@ class EventStoreProjectionTest { // Give the projection time to process the event. delay(150) - assertSame(seed, projection.items.value) + assertSame(seed, projection.items) assertEquals( a.id, - projection.items.value[0] + projection.items[0] .value.id, ) projection.close() @@ -299,8 +309,8 @@ class EventStoreProjectionTest { observable.insert(b) val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() - assertEquals(2, projection.items.value.size) + projection.awaitReady() + assertEquals(2, projection.items.size) val vanish = signer.sign( @@ -328,8 +338,8 @@ class EventStoreProjectionTest { observable.insert(a) val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() - val seed = projection.items.value + projection.awaitReady() + val seed = projection.items val foreignVanish = otherSigner.sign( @@ -341,7 +351,7 @@ class EventStoreProjectionTest { observable.insert(foreignVanish) delay(150) - assertSame(seed, projection.items.value) + assertSame(seed, projection.items) projection.close() } @@ -364,8 +374,8 @@ class EventStoreProjectionTest { Filter(kinds = listOf(TextNoteEvent.KIND)), scope, ) - projection.ready.await() - assertEquals(2, projection.items.value.size) + projection.awaitReady() + assertEquals(2, projection.items.size) // Let the short expiration lapse, then ask the store to // sweep — the projection drops the expired slot in @@ -398,8 +408,8 @@ class EventStoreProjectionTest { Filter(kinds = listOf(TextNoteEvent.KIND)), scope, ) - projection.ready.await() - assertEquals(3, projection.items.value.size) + projection.awaitReady() + assertEquals(3, projection.items.size) // Drop everything authored by `signer` — should leave // only the foreign event. @@ -423,8 +433,8 @@ class EventStoreProjectionTest { Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), scope, ) - projection.ready.await() - assertEquals(2, projection.items.value.size) + projection.awaitReady() + assertEquals(2, projection.items.size) val c = signer.sign(TextNoteEvent.build("c", createdAt = 300)) observable.insert(c) @@ -462,8 +472,8 @@ class EventStoreProjectionTest { listOf(filterA, filterB), scope, ) - projection.ready.await() - assertEquals(4, projection.items.value.size, "per-filter caps don't dedupe union") + projection.awaitReady() + assertEquals(4, projection.items.size, "per-filter caps don't dedupe union") projection.close() } @@ -485,8 +495,8 @@ class EventStoreProjectionTest { Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), scope, ) - projection.ready.await() - assertEquals(2, projection.items.value.size) + projection.awaitReady() + assertEquals(2, projection.items.size) observable.insert(a3) val after = projection.awaitItems { it[0].value.id == a3.id } @@ -503,12 +513,12 @@ class EventStoreProjectionTest { observable.insert(a) val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.ready.await() + projection.awaitReady() projection.close() observable.insert(signer.sign(TextNoteEvent.build("b", createdAt = 200))) delay(150) - assertTrue(projection.items.value.isEmpty()) + assertTrue(projection.items.isEmpty()) } /** @@ -525,8 +535,8 @@ class EventStoreProjectionTest { val ephemeralKind = 22_000 val projection = observable.observe(Filter(kinds = listOf(ephemeralKind)), scope) - projection.ready.await() - assertTrue(projection.items.value.isEmpty()) + projection.awaitReady() + assertTrue(projection.items.isEmpty()) val ephemeral: Event = signer.sign( @@ -548,8 +558,8 @@ class EventStoreProjectionTest { // event was only ever live, not durable. val freshProjection = observable.observe(Filter(kinds = listOf(ephemeralKind)), scope) - freshProjection.ready.await() - assertTrue(freshProjection.items.value.isEmpty()) + freshProjection.awaitReady() + assertTrue(freshProjection.items.isEmpty()) projection.close() freshProjection.close() From f789fe4f53722506eec833302120a5e2d80fd205 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 15:35:30 +0000 Subject: [PATCH 18/24] refactor(quartz): EventStoreProjection becomes pure state machine; ObservableEventStore.project returns cold Flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes scope ownership from EventStoreProjection. The class is now a pure state machine — no Job, no AutoCloseable, no embedded StateFlow: class EventStoreProjection(store, filters, nowProvider) { suspend fun seed() // run the seed query fun apply(change: StoreChange): Boolean // apply one mutation fun snapshot(): ProjectionState.Loaded // current view } ObservableEventStore.observe(filter, scope) is renamed to ObservableEventStore.project(filter) and now returns a cold Flow>. Each collection allocates its own state machine, runs its own seed, and unsubscribes when the collector cancels. The flow is built using channelFlow + the state machine — the class stays as the load-bearing primitive. Standard caller pattern: val state = observable.project(filter) .stateIn(viewModelScope, SharingStarted.WhileSubscribed(5_000), ProjectionState.Loading) Tradeoffs: - Cold flow → multiple collectors each re-seed. ViewModel pattern with stateIn shares one collection across observers (the standard Android approach). - close() goes away. Cancelling the collecting scope tears down the projection. - closeStopsListening test renamed to cancellingScopeStopsListening and now verifies the StateFlow's value freezes after its scope is cancelled. 17/17 projection + 240/240 store tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../cache/projection/EventStoreProjection.kt | 255 +++++++++--------- .../projection/EventStoreProjectionTest.kt | 148 +++++----- 2 files changed, 185 insertions(+), 218 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt index 40e07a416..60cccf113 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt @@ -33,17 +33,14 @@ import com.vitorpamplona.quartz.nip40Expiration.isExpirationBefore import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.TimeUtils -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.onSubscription -import kotlinx.coroutines.launch import kotlinx.coroutines.yield /** - * Lifecycle state of an [EventStoreProjection]'s [state][EventStoreProjection.state]. + * Lifecycle state of an [ObservableEventStore.project] flow. * * - [Loading] is the initial state before the seed query completes. * The UI should show a spinner / skeleton here. @@ -60,31 +57,28 @@ sealed interface ProjectionState { } /** - * A reactive projection over an [ObservableEventStore] for a fixed - * set of [filters]. Each visible event is wrapped in a - * [MutableStateFlow] so the UI can collect three different kinds of - * change with the right granularity: + * State machine that maintains a reactive view over an + * [ObservableEventStore] for a fixed set of [filters]. Each visible + * event is wrapped in a [MutableStateFlow] so the UI can collect + * three different kinds of change with the right granularity: * - * - **Membership** (events arriving or leaving) re-emits a brand new - * [List] from [items]. The list reference is stable while membership - * is unchanged. + * - **Membership** (events arriving or leaving) produces a brand + * new [List] in [snapshot]. The list reference is stable while + * membership is unchanged. * - **In-place replaceable / addressable update** (a new version of - * the same `kind:pubkey:dtag` arrives) updates the existing handle's - * [MutableStateFlow.value] without touching the list. Only collectors - * of that one handle re-render. The list ordering is *not* - * reshuffled when the new version has a later `created_at` — each - * slot remembers the sort key it was inserted with, so updates feel - * like pure value mutations. + * the same `kind:pubkey:dtag` arrives) updates the existing + * handle's [MutableStateFlow.value] without changing membership. + * Only collectors of that one handle re-render. The list ordering + * is *not* reshuffled when the new version has a later + * `created_at` — each slot remembers the sort key it was inserted + * with, so updates feel like pure value mutations. * - **Removal** (NIP-09 deletion, NIP-62 vanish, NIP-40 expiration, * `delete(filter)`) drops the handle from the list. * - * The seed is materialised by querying the store once at start, after - * which the projection is driven entirely by - * [ObservableEventStore.changes]. Three kinds of mutation arrive on - * that stream: + * Three kinds of [StoreChange] are interpreted in-projection: * - * - [StoreChange.Insert] — interpreted in-projection so a single - * arriving event can carry NIP-01 / NIP-09 / NIP-62 semantics: + * - [StoreChange.Insert] — a single arriving event can carry NIP-01 + * / NIP-09 / NIP-62 semantics: * - **NIP-01 supersession.** New replaceable / addressable events * replace prior ones for the same `kind:pubkey[:dtag]`. The * NIP-01 lexical-id tiebreaker (`new.id < old.id` when @@ -97,55 +91,43 @@ sealed interface ProjectionState { * from the same author with `created_at < vanish.created_at`. * - **NIP-40 expiration.** Events whose `expiration` tag has * already lapsed at the moment they arrive are dropped before - * they ever enter [items]. - * + * they ever enter the snapshot. * - [StoreChange.DeleteByFilter] — emitted on `delete(filter)` / - * `delete(filters)`. The projection drops every slot matching any - * of the rule's filters via [Filter.match]. - * + * `delete(filters)`. Drops every slot matching any of the rule's + * filters via [Filter.match]. * - [StoreChange.DeleteExpired] — emitted on `deleteExpiredEvents()`. - * The projection drops every slot whose `expiration` has lapsed at - * the cutoff the store pinned. **There is no per-projection - * expiration ticker** — projections only drop expired events when - * the application calls `deleteExpiredEvents()` on the store. + * Drops every slot whose `expiration` has lapsed at the cutoff the + * store pinned. **There is no per-projection expiration ticker** — + * expiration only triggers when the application calls + * `deleteExpiredEvents()` on the store. * * Limit handling is **per-filter**: each filter retains at most its * own `limit` matches in a private capped set, sorted by created_at - * DESC + id ASC. The projection's [items] is the deduped union of - * those sets, so when filter A and filter B match disjoint events the - * union can be larger than any single filter's `limit`. We do not - * refill from the store after a deletion — if a removal leaves a - * filter under cap, it stays under cap until another match arrives. + * DESC + id ASC. The snapshot is the deduped union of those sets, so + * when filter A and filter B match disjoint events the union can be + * larger than any single filter's `limit`. We do not refill from the + * store after a deletion — if a removal leaves a filter under cap, it + * stays under cap until another match arrives. * - * Ephemeral events (kinds `20000-29999`) reach the projection via - * [ObservableEventStore.changes] without ever being persisted; they - * appear in [items] for as long as the projection is alive but never - * survive a re-seed. They aren't covered by the store's - * `deleteExpiredEvents()` sweep (the DB never had them), so an - * ephemeral with an `expiration` tag will linger in the projection + * Ephemeral events (kinds `20000-29999`) reach the projection without + * ever being persisted; they appear in the snapshot for as long as + * the projection is alive but never survive a re-seed. They aren't + * covered by the store's `deleteExpiredEvents()` sweep (the DB never + * had them), so an ephemeral with an `expiration` tag will linger * until it's superseded or until the projection is closed. * - * Lifecycle: the projection runs a single collector inside [scope]. - * Cancel the scope (or call [close]) when the screen using the - * projection goes away. + * **Lifecycle**: this class is a pure state machine with no + * coroutine ownership. Construct it, call [seed] once, then call + * [apply] for each [StoreChange] from [ObservableEventStore.changes]. + * For the standard "wrap into a Flow and collect from a ViewModel" + * use case, use the [ObservableEventStore.project] extension which + * does this composition for you. */ class EventStoreProjection( private val store: ObservableEventStore, private val filters: List, - scope: CoroutineScope, private val nowProvider: () -> Long = TimeUtils::now, -) : AutoCloseable { - private val _state = MutableStateFlow>(ProjectionState.Loading) - - /** - * Current projection state. Starts as [ProjectionState.Loading] - * until the seed query completes; thereafter every membership - * change publishes a fresh [ProjectionState.Loaded] with the new - * list. Distinguishing `Loading` from `Loaded(emptyList())` lets - * the UI tell "still seeding" apart from "seeded but no matches". - */ - val state: StateFlow> = _state.asStateFlow() - +) { /** Slots keyed by the *current* event id. Re-keyed when a replaceable / addressable handle takes a new version. */ private val byId = HashMap>() @@ -161,51 +143,56 @@ class EventStoreProjection( private val perFilter: Map>> = filters.associateWith { sortedSetOf(slotComparator()) } - private val collectorJob: Job = - scope.launch { - // `onSubscription` runs after the SharedFlow subscription is - // active but before we pull any events — the buffer absorbs - // emissions arriving during seed and drains them once collect - // proceeds. Doing the seed inside `collect { }` would race - // with concurrent inserts. - store.changes - .onSubscription { - seed() - }.collect { storeEvent -> apply(storeEvent) } - } - - private suspend fun seed() { + /** + * Run the seed query against the store and populate the indexes. + * Call once, before the first [apply]. Expired events are skipped + * (the store doesn't filter them at query time, so we do here). + */ + suspend fun seed() { val now = nowProvider() for (event in store.query(filters)) { - // Stores don't filter expired rows at query time, so do it - // here — otherwise an expired event would briefly appear - // in [items] before the next deleteExpiredEvents() sweep. if (event.isExpirationBefore(now)) continue applyInsert(event) yield() } - publish() } - private fun apply(storeEvent: StoreChange) { - val changed = - when (storeEvent) { - is StoreChange.Insert -> { - applyInsert(storeEvent.event) - } - - is StoreChange.DeleteByFilter -> { - dropWhere { ev -> storeEvent.filters.any { it.match(ev) } } - } - - // Store's sweep uses strict `<`; isExpirationBefore is - // `<=`, so subtract 1 to match. - is StoreChange.DeleteExpired -> { - val cutoff = (storeEvent.asOf ?: nowProvider()) - 1 - dropWhere { it.isExpirationBefore(cutoff) } - } + /** + * Apply a [StoreChange] from [ObservableEventStore.changes]. + * Returns true if the change altered membership (caller should + * publish a fresh [snapshot]); false for in-place addressable + * updates, NIP-01 tiebreaker rejections, and arrivals that no + * filter matches. + */ + fun apply(storeEvent: StoreChange): Boolean = + when (storeEvent) { + is StoreChange.Insert -> { + applyInsert(storeEvent.event) } - if (changed) publish() + + is StoreChange.DeleteByFilter -> { + dropWhere { ev -> storeEvent.filters.any { it.match(ev) } } + } + + // Store's sweep uses strict `<`; isExpirationBefore is + // `<=`, so subtract 1 to match. + is StoreChange.DeleteExpired -> { + val cutoff = (storeEvent.asOf ?: nowProvider()) - 1 + dropWhere { it.isExpirationBefore(cutoff) } + } + } + + /** + * Current snapshot as a [ProjectionState.Loaded]. Cheap — the + * deduped sorted union over [perFilter] sets is computed on + * demand. Returns an empty [ProjectionState.Loaded] when no slots + * are live. + */ + fun snapshot(): ProjectionState.Loaded { + if (byId.isEmpty()) return ProjectionState.Loaded(emptyList()) + val union = sortedSetOf(slotComparator()) + for (set in perFilter.values) union.addAll(set) + return ProjectionState.Loaded(union.map { it.flow }) } private fun applyInsert(event: Event): Boolean { @@ -341,32 +328,6 @@ class EventStoreProjection( return true } - private fun publish() { - // Lazily compute the deduped sorted union from per-filter - // sets. Cheaper than maintaining a separate `ordered` field - // alongside every insert / remove. - if (byId.isEmpty()) { - _state.value = ProjectionState.Loaded(emptyList()) - return - } - val union = sortedSetOf(slotComparator()) - for (set in perFilter.values) union.addAll(set) - _state.value = ProjectionState.Loaded(union.map { it.flow }) - } - - /** - * Stop tracking changes and clear internal state. Idempotent. The - * scope passed to the constructor keeps running; only this - * projection's collector job is cancelled. - */ - override fun close() { - collectorJob.cancel() - byId.clear() - byAddress.clear() - for (set in perFilter.values) set.clear() - _state.value = ProjectionState.Loaded(emptyList()) - } - /** * Internal slot. Each event added to the projection lives inside * one of these for as long as it survives. The sort key is frozen @@ -430,17 +391,41 @@ class EventStoreProjection( } /** - * Convenience: open a projection over this observable store. NIP-62 - * vanish handling is scoped by the inner store's `relay`. Cancel - * [scope] (or call [EventStoreProjection.close]) to release the - * projection. + * Open a cold reactive projection over this observable store for the + * given [filters]. The returned flow: + * + * - emits [ProjectionState.Loading] on subscription; + * - runs the seed query against the store; + * - emits [ProjectionState.Loaded] with the seeded list; + * - then emits a fresh [ProjectionState.Loaded] every time membership + * changes (in-place addressable updates do not re-emit — collectors + * of the slot's own [MutableStateFlow] handle those). + * + * Each collection allocates its own state machine, runs its own seed, + * and unsubscribes when the collector cancels — there's no + * `close()`. For multiple collectors on the same view, share with + * `stateIn(scope, SharingStarted.WhileSubscribed(...), Loading)` (the + * standard ViewModel pattern). + * + * NIP-62 vanish handling is scoped by the inner store's `relay`. */ -fun ObservableEventStore.observe( - filters: List, - scope: CoroutineScope, -): EventStoreProjection = EventStoreProjection(this, filters, scope) +fun ObservableEventStore.project(filters: List): Flow> = + channelFlow { + val projection = EventStoreProjection(this@project, filters) + send(ProjectionState.Loading) + // `onSubscription` runs after the SharedFlow subscription is + // active but before we pull events — the buffer absorbs + // emissions arriving during the seed query and we drain them + // through `apply` once collect proceeds. Doing the seed + // inside `collect { }` instead would race with concurrent + // inserts. + changes + .onSubscription { + projection.seed() + send(projection.snapshot()) + }.collect { change -> + if (projection.apply(change)) send(projection.snapshot()) + } + } -fun ObservableEventStore.observe( - filter: Filter, - scope: CoroutineScope, -): EventStoreProjection = EventStoreProjection(this, listOf(filter), scope) +fun ObservableEventStore.project(filter: Filter): Flow> = project(listOf(filter)) diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt index 4112b908b..68a98e28b 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt @@ -39,7 +39,10 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout import kotlin.test.AfterTest @@ -71,22 +74,27 @@ class EventStoreProjectionTest { store.close() } + /** Open a hot StateFlow over the projection for the lifetime of the test scope. */ + private fun projectionOf(filter: Filter): StateFlow> = observable.project(filter).stateIn(scope, SharingStarted.Eagerly, ProjectionState.Loading) + + private fun projectionOf(filters: List): StateFlow> = observable.project(filters).stateIn(scope, SharingStarted.Eagerly, ProjectionState.Loading) + /** Snapshot of the currently-loaded items, or empty if still seeding. */ - private val EventStoreProjection.items: List> - get() = (state.value as? ProjectionState.Loaded)?.items.orEmpty() + private val StateFlow>.items: List> + get() = (value as? ProjectionState.Loaded)?.items.orEmpty() /** Suspends until the seed completes; returns the loaded list. */ - private suspend fun EventStoreProjection.awaitReady(timeoutMs: Long = 5_000): List> = + private suspend fun StateFlow>.awaitLoaded(timeoutMs: Long = 5_000): List> = withTimeout(timeoutMs) { - (state.first { it is ProjectionState.Loaded } as ProjectionState.Loaded).items + (first { it is ProjectionState.Loaded } as ProjectionState.Loaded).items } - private suspend fun EventStoreProjection.awaitItems( + private suspend fun StateFlow>.awaitItems( timeoutMs: Long = 5_000, predicate: (List>) -> Boolean, ): List> = withTimeout(timeoutMs) { - (state.first { it is ProjectionState.Loaded && predicate(it.items) } as ProjectionState.Loaded).items + (first { it is ProjectionState.Loaded && predicate(it.items) } as ProjectionState.Loaded).items } private suspend fun awaitFlow( @@ -106,14 +114,13 @@ class EventStoreProjectionTest { observable.insert(a) observable.insert(b) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.awaitReady() + val projection = projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() val items = projection.items assertEquals(2, items.size) assertEquals(b.id, items[0].value.id) assertEquals(a.id, items[1].value.id) - projection.close() } @Test @@ -122,8 +129,8 @@ class EventStoreProjectionTest { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) observable.insert(a) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.awaitReady() + val projection = projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() val before = projection.items assertEquals(1, before.size) @@ -133,7 +140,6 @@ class EventStoreProjectionTest { val after = projection.awaitItems { it.size == 2 } assertNotSame(before, after, "insert must produce a new list reference") assertEquals(b.id, after[0].value.id) - projection.close() } @Test @@ -142,8 +148,8 @@ class EventStoreProjectionTest { val text = signer.sign(TextNoteEvent.build("a", createdAt = 100)) observable.insert(text) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.awaitReady() + val projection = projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() val seed = projection.items val meta = signer.sign(MetadataEvent.createNew("Vitor", createdAt = 200)) @@ -151,7 +157,6 @@ class EventStoreProjectionTest { delay(150) assertSame(seed, projection.items) - projection.close() } @Test @@ -162,11 +167,8 @@ class EventStoreProjectionTest { observable.insert(v1) val projection = - observable.observe( - Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), - scope, - ) - projection.awaitReady() + projectionOf(Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey))) + projection.awaitLoaded() val seedList = projection.items assertEquals(1, seedList.size) val slot = seedList[0] @@ -178,7 +180,6 @@ class EventStoreProjectionTest { awaitFlow(slot) { it.id == v2.id } assertSame(seedList, projection.items, "replaceable update must not change list reference") assertSame(slot, projection.items[0]) - projection.close() } @Test @@ -189,15 +190,14 @@ class EventStoreProjectionTest { observable.insert(v1) val projection = - observable.observe( + projectionOf( Filter( kinds = listOf(LongTextNoteEvent.KIND), authors = listOf(v1.pubKey), tags = mapOf("d" to listOf("blog")), ), - scope, ) - projection.awaitReady() + projection.awaitLoaded() val seedList = projection.items val slot = seedList[0] @@ -206,7 +206,6 @@ class EventStoreProjectionTest { awaitFlow(slot) { it.id == v2.id } assertSame(seedList, projection.items, "addressable update must not change list reference") - projection.close() } /** @@ -228,11 +227,8 @@ class EventStoreProjectionTest { observable.insert(v2) val projection = - observable.observe( - Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey)), - scope, - ) - projection.awaitReady() + projectionOf(Filter(kinds = listOf(MetadataEvent.KIND), authors = listOf(v1.pubKey))) + projection.awaitLoaded() val slot = projection.items[0] assertEquals(v2.id, slot.value.id) @@ -247,7 +243,6 @@ class EventStoreProjectionTest { delay(150) assertEquals(v2.id, slot.value.id) - projection.close() } @Test @@ -258,8 +253,8 @@ class EventStoreProjectionTest { observable.insert(a) observable.insert(b) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.awaitReady() + val projection = projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() assertEquals(2, projection.items.size) val deletion = signer.sign(DeletionEvent.build(listOf(a))) @@ -267,7 +262,6 @@ class EventStoreProjectionTest { val after = projection.awaitItems { it.size == 1 } assertEquals(b.id, after[0].value.id) - projection.close() } /** @@ -280,8 +274,8 @@ class EventStoreProjectionTest { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) observable.insert(a) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.awaitReady() + val projection = projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() val seed = projection.items assertEquals(1, seed.size) @@ -296,7 +290,6 @@ class EventStoreProjectionTest { projection.items[0] .value.id, ) - projection.close() } @Test @@ -308,8 +301,8 @@ class EventStoreProjectionTest { observable.insert(a) observable.insert(b) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.awaitReady() + val projection = projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() assertEquals(2, projection.items.size) val vanish = @@ -323,7 +316,6 @@ class EventStoreProjectionTest { val after = projection.awaitItems { it.isEmpty() } assertTrue(after.isEmpty()) - projection.close() } /** @@ -337,8 +329,8 @@ class EventStoreProjectionTest { val a = signer.sign(TextNoteEvent.build("a", createdAt = time)) observable.insert(a) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.awaitReady() + val projection = projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() val seed = projection.items val foreignVanish = @@ -352,7 +344,6 @@ class EventStoreProjectionTest { delay(150) assertSame(seed, projection.items) - projection.close() } /** @@ -370,11 +361,8 @@ class EventStoreProjectionTest { observable.insert(short) val projection = - observable.observe( - Filter(kinds = listOf(TextNoteEvent.KIND)), - scope, - ) - projection.awaitReady() + projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() assertEquals(2, projection.items.size) // Let the short expiration lapse, then ask the store to @@ -385,7 +373,6 @@ class EventStoreProjectionTest { val after = projection.awaitItems { it.size == 1 } assertEquals(safe.id, after[0].value.id) - projection.close() } /** @@ -404,11 +391,8 @@ class EventStoreProjectionTest { observable.insert(foreign) val projection = - observable.observe( - Filter(kinds = listOf(TextNoteEvent.KIND)), - scope, - ) - projection.awaitReady() + projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND))) + projection.awaitLoaded() assertEquals(3, projection.items.size) // Drop everything authored by `signer` — should leave @@ -417,7 +401,6 @@ class EventStoreProjectionTest { val after = projection.awaitItems { it.size == 1 } assertEquals(foreign.id, after[0].value.id) - projection.close() } @Test @@ -429,11 +412,8 @@ class EventStoreProjectionTest { observable.insert(b) val projection = - observable.observe( - Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), - scope, - ) - projection.awaitReady() + projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2)) + projection.awaitLoaded() assertEquals(2, projection.items.size) val c = signer.sign(TextNoteEvent.build("c", createdAt = 300)) @@ -443,7 +423,6 @@ class EventStoreProjectionTest { assertEquals(2, after.size) assertEquals(c.id, after[0].value.id) assertEquals(b.id, after[1].value.id) - projection.close() } /** @@ -468,13 +447,11 @@ class EventStoreProjectionTest { val filterA = Filter(kinds = listOf(TextNoteEvent.KIND), authors = listOf(authorA.pubKey), limit = 2) val filterB = Filter(kinds = listOf(TextNoteEvent.KIND), authors = listOf(authorB.pubKey), limit = 2) val projection = - observable.observe( + projectionOf( listOf(filterA, filterB), - scope, ) - projection.awaitReady() + projection.awaitLoaded() assertEquals(4, projection.items.size, "per-filter caps don't dedupe union") - projection.close() } /** @@ -491,11 +468,8 @@ class EventStoreProjectionTest { observable.insert(a2) val projection = - observable.observe( - Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2), - scope, - ) - projection.awaitReady() + projectionOf(Filter(kinds = listOf(TextNoteEvent.KIND), limit = 2)) + projection.awaitLoaded() assertEquals(2, projection.items.size) observable.insert(a3) @@ -503,22 +477,33 @@ class EventStoreProjectionTest { assertEquals(2, after.size) assertEquals(a3.id, after[0].value.id) assertEquals(a2.id, after[1].value.id) - projection.close() } @Test - fun closeStopsListening() = + fun cancellingScopeStopsListening() = runBlocking { val a = signer.sign(TextNoteEvent.build("a", createdAt = 100)) observable.insert(a) - val projection = observable.observe(Filter(kinds = listOf(TextNoteEvent.KIND)), scope) - projection.awaitReady() - projection.close() + // Sub-scope so we can cancel just the projection's collector + // without taking down the test's outer scope. + val collectorScope = CoroutineScope(SupervisorJob()) + val projection = + observable + .project(Filter(kinds = listOf(TextNoteEvent.KIND))) + .stateIn(collectorScope, SharingStarted.Eagerly, ProjectionState.Loading) + projection.awaitLoaded() + assertEquals(1, projection.items.size) + collectorScope.cancel() + delay(50) + + // After the collector scope dies, new inserts don't update + // the StateFlow — its value is frozen at the last emission. observable.insert(signer.sign(TextNoteEvent.build("b", createdAt = 200))) delay(150) - assertTrue(projection.items.isEmpty()) + assertEquals(1, projection.items.size) + assertEquals(a.id, projection.items[0].value.id) } /** @@ -534,8 +519,8 @@ class EventStoreProjectionTest { runBlocking { val ephemeralKind = 22_000 val projection = - observable.observe(Filter(kinds = listOf(ephemeralKind)), scope) - projection.awaitReady() + projectionOf(Filter(kinds = listOf(ephemeralKind))) + projection.awaitLoaded() assertTrue(projection.items.isEmpty()) val ephemeral: Event = @@ -557,11 +542,8 @@ class EventStoreProjectionTest { // A fresh projection on the same store gets nothing — the // event was only ever live, not durable. val freshProjection = - observable.observe(Filter(kinds = listOf(ephemeralKind)), scope) - freshProjection.awaitReady() + projectionOf(Filter(kinds = listOf(ephemeralKind))) + freshProjection.awaitLoaded() assertTrue(freshProjection.items.isEmpty()) - - projection.close() - freshProjection.close() } } From 5755ab90b20ea21e4c46440117b26a9b1c976715 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 15:48:36 +0000 Subject: [PATCH 19/24] refactor(quartz): switch project() builder from channelFlow to flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit channelFlow was overkill for a single-producer chain. The cost was an extra Channel hop between every projection.snapshot() and the downstream collector — pure overhead with no concurrency benefit. flow { } needs the outer collector captured (because inside changes.onSubscription { ... } and changes.collect { ... } the implicit `this` is FlowCollector, not FlowCollector>). One `val outer = this` fixes that. Backpressure now flows directly: a slow collector suspends emit, which suspends our changes.collect, which suspends the SharedFlow's buffer drain — natural propagation, no decoupling Channel in the middle. 17/17 projection tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../cache/projection/EventStoreProjection.kt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt index 60cccf113..bcd43f082 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt @@ -35,7 +35,7 @@ import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.onSubscription import kotlinx.coroutines.yield @@ -410,9 +410,13 @@ class EventStoreProjection( * NIP-62 vanish handling is scoped by the inner store's `relay`. */ fun ObservableEventStore.project(filters: List): Flow> = - channelFlow { + flow { val projection = EventStoreProjection(this@project, filters) - send(ProjectionState.Loading) + // Capture the outer collector so we can emit ProjectionState + // from inside `changes.onSubscription { }` and `collect { }`, + // where the implicit `this` is FlowCollector. + val outer = this + emit(ProjectionState.Loading) // `onSubscription` runs after the SharedFlow subscription is // active but before we pull events — the buffer absorbs // emissions arriving during the seed query and we drain them @@ -422,9 +426,9 @@ fun ObservableEventStore.project(filters: List): Flow - if (projection.apply(change)) send(projection.snapshot()) + if (projection.apply(change)) outer.emit(projection.snapshot()) } } From f4b94d6c6a06ab5b614ab2c5c45c6bea6c648a46 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 15:52:21 +0000 Subject: [PATCH 20/24] feat(quartz): InterningEventStore now interns accepted events on insert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the inner store accepts an event (insert succeeds), register the caller's instance with the interner so subsequent reads of the same id return the same `===` reference. This means an event that was just inserted and is then queried back resolves to the original in-memory object, not a freshly-deserialized clone. Same in transaction { }: every accepted event is interned after the inner transaction commits. If the body throws or inner rolls back, nothing is interned (accepted list is discarded). The decorator still does not *substitute* the caller's event with a previously-cached one — same-id-different-sig collisions resolve the new event into the cache without overwriting (intern() is first-seen-wins). The caller keeps the instance they passed in. All cache + store + projection tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../cache/interning/InterningEventStore.kt | 45 ++++++++++++++++--- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt index 3e521ee22..d224a17d9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/InterningEventStore.kt @@ -39,11 +39,20 @@ import com.vitorpamplona.quartz.nip01Core.store.IEventStore * val observable = ObservableEventStore(cached) * ``` * - * Writes (`insert`, `transaction`, `delete*`) pass through unchanged - * — the decorator never substitutes the caller's event for a cached - * one on the way in (sigs may differ across "same id, different - * decode" cases). Canonicalisation only happens on results coming - * back out of the store. + * Writes ([insert] / [transaction]) forward to the inner store and, + * on success, register the accepted event with the interner. The + * caller's instance becomes canonical for that id (so a later read + * of the same id returns the same `===` reference, until the weak + * ref is collected). The decorator does *not* substitute the + * caller's event for a previously-cached one on the way in — sigs + * may differ across "same id, different decode" cases, and the + * caller's instance is the freshest. + * + * Reads ([query]) pipe results through `interner.intern`, returning + * canonical instances for ids that have a live cached entry. + * + * Out-of-band removals (`delete*`, `deleteExpiredEvents`) pass + * through unchanged. * * The default [interner] is [EventInterner.Default]; pass a fresh * instance for tests or any context that needs isolation. @@ -54,9 +63,31 @@ class InterningEventStore( ) : IEventStore { override val relay: NormalizedRelayUrl? get() = inner.relay - override suspend fun insert(event: Event) = inner.insert(event) + override suspend fun insert(event: Event) { + inner.insert(event) + // Inner accepted (didn't throw) — register the canonical + // instance so subsequent reads share the same reference. + interner.intern(event) + } - override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) = inner.transaction(body) + override suspend fun transaction(body: IEventStore.ITransaction.() -> Unit) { + // Capture every accepted event so we can intern after the + // inner transaction commits. If body throws or the inner + // rolls back, `accepted` is discarded with the txn. + val accepted = ArrayList() + inner.transaction { + val innerTxn = this + val wrapped = + object : IEventStore.ITransaction { + override fun insert(event: Event) { + innerTxn.insert(event) + accepted.add(event) + } + } + wrapped.body() + } + for (e in accepted) interner.intern(e) + } @Suppress("UNCHECKED_CAST") override suspend fun query(filter: Filter): List = inner.query(filter).map { interner.intern(it) as T } From f759f44eeaf000b0de67e085da619d9d5f4ce27d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 16:03:50 +0000 Subject: [PATCH 21/24] docs(quartz): trim verbose KDoc on cache + observable layers Reviews before merge: - EventInterner: drop stale "used by Event.fromJson" reference (we reverted that earlier). Tighten the class doc. - ObservableEventStore: collapse the 30+ line class KDoc + duplicated StoreChange description into one focused doc each. - EventStoreProjection: trim the 60+ line class doc; reactive semantics live on the project() extension. Add explicit caveat about in-place addressable updates not re-evaluating filter membership. No behaviour change. All cache + store + projection tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../cache/interning/EventInterner.kt | 28 ++---- .../cache/projection/EventStoreProjection.kt | 88 ++++++------------- .../nip01Core/store/ObservableEventStore.kt | 67 ++++---------- 3 files changed, 52 insertions(+), 131 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.kt index fe0d02005..d0f206645 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/interning/EventInterner.kt @@ -24,31 +24,21 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey /** - * Process-wide interner that canonicalises [Event] instances by id so - * every consumer (relay client, store deserialization, projections, - * tests) sees the same object reference for the same event id. + * Canonicalises [Event] instances by id so every consumer sees the + * same object reference for the same event id. * * Backed by weak references — entries vanish when no live consumer - * holds the event, so the cache only grows as long as projections / - * UI state actually need the events. Sized for ~5000 hot events; the - * underlying map resizes if usage exceeds that. - * - * Use [intern] on every event arrival path. The first occurrence wins - * and becomes canonical; subsequent equivalent decodes return that - * canonical instance. + * holds the event. First-seen wins; equivalence is by event id, so + * callers must trust ids are content-derived (signed events satisfy + * this). Sized for ~5000 hot entries; resizes if usage exceeds that. * * Platforms without weak references fall back to a passthrough that - * returns [event] unchanged — no canonicalisation, but no leaks - * either. + * returns [event] unchanged — no canonicalisation, no leaks. */ expect class EventInterner() { /** - * Returns the canonical [Event] for [event]'s id. If a live - * canonical instance already exists for this id, returns it; - * otherwise stores [event] as the new canonical and returns it. - * - * Equivalence is by event id only — callers must trust the id - * was content-derived (signed events satisfy this). + * Returns the canonical [Event] for [event]'s id, storing + * [event] as canonical if no live entry exists. */ fun intern(event: Event): Event @@ -62,7 +52,7 @@ expect class EventInterner() { fun clear() companion object { - /** Process-wide default interner used by [Event.fromJson]. */ + /** Process-wide shared instance. */ val Default: EventInterner } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt index bcd43f082..31092bd34 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt @@ -58,70 +58,35 @@ sealed interface ProjectionState { /** * State machine that maintains a reactive view over an - * [ObservableEventStore] for a fixed set of [filters]. Each visible - * event is wrapped in a [MutableStateFlow] so the UI can collect - * three different kinds of change with the right granularity: + * [ObservableEventStore] for a fixed set of [filters]. Pure logic, no + * coroutine ownership: construct, [seed] once, then [apply] each + * [StoreChange] and read [snapshot]. For the standard "wrap into a + * cold Flow and collect from a ViewModel" path, use + * [ObservableEventStore.project] which composes this class. * - * - **Membership** (events arriving or leaving) produces a brand - * new [List] in [snapshot]. The list reference is stable while - * membership is unchanged. - * - **In-place replaceable / addressable update** (a new version of - * the same `kind:pubkey:dtag` arrives) updates the existing - * handle's [MutableStateFlow.value] without changing membership. - * Only collectors of that one handle re-render. The list ordering - * is *not* reshuffled when the new version has a later - * `created_at` — each slot remembers the sort key it was inserted - * with, so updates feel like pure value mutations. - * - **Removal** (NIP-09 deletion, NIP-62 vanish, NIP-40 expiration, - * `delete(filter)`) drops the handle from the list. + * Each visible event is wrapped in a [MutableStateFlow], giving the + * UI three change granularities: * - * Three kinds of [StoreChange] are interpreted in-projection: + * - **Membership** (insert / removal): [snapshot] returns a fresh + * list. Stable list reference while membership is unchanged. + * - **In-place replaceable / addressable update**: same slot, new + * [MutableStateFlow.value]. Only collectors of that handle + * re-render. The slot's sort key is frozen at insertion, so the + * list does not reshuffle when the new version has a later + * `created_at`. *Caveat: the slot stays in whatever filters + * matched the original version; if the new version no longer + * matches a filter (e.g. its tags changed), the slot is not + * re-evaluated and remains live.* + * - **Removal**: NIP-09, NIP-62, NIP-40 expiration, `delete(filter)`. * - * - [StoreChange.Insert] — a single arriving event can carry NIP-01 - * / NIP-09 / NIP-62 semantics: - * - **NIP-01 supersession.** New replaceable / addressable events - * replace prior ones for the same `kind:pubkey[:dtag]`. The - * NIP-01 lexical-id tiebreaker (`new.id < old.id` when - * `created_at` ties) is honoured. - * - **NIP-09 deletions.** A [DeletionEvent] removes any matching - * handle owned by the same author (for GiftWrap, the recipient). - * Cross-author deletions are inert. - * - **NIP-62 right to vanish.** A [RequestToVanishEvent] whose - * `shouldVanishFrom(store.relay)` is true drops every handle - * from the same author with `created_at < vanish.created_at`. - * - **NIP-40 expiration.** Events whose `expiration` tag has - * already lapsed at the moment they arrive are dropped before - * they ever enter the snapshot. - * - [StoreChange.DeleteByFilter] — emitted on `delete(filter)` / - * `delete(filters)`. Drops every slot matching any of the rule's - * filters via [Filter.match]. - * - [StoreChange.DeleteExpired] — emitted on `deleteExpiredEvents()`. - * Drops every slot whose `expiration` has lapsed at the cutoff the - * store pinned. **There is no per-projection expiration ticker** — - * expiration only triggers when the application calls - * `deleteExpiredEvents()` on the store. + * Limit is **per-filter**: each filter retains at most its own + * `limit` matches; the snapshot is the deduped union, so disjoint + * matches across filters can exceed any single cap. Removed slots + * are not refilled from the store. * - * Limit handling is **per-filter**: each filter retains at most its - * own `limit` matches in a private capped set, sorted by created_at - * DESC + id ASC. The snapshot is the deduped union of those sets, so - * when filter A and filter B match disjoint events the union can be - * larger than any single filter's `limit`. We do not refill from the - * store after a deletion — if a removal leaves a filter under cap, it - * stays under cap until another match arrives. - * - * Ephemeral events (kinds `20000-29999`) reach the projection without - * ever being persisted; they appear in the snapshot for as long as - * the projection is alive but never survive a re-seed. They aren't - * covered by the store's `deleteExpiredEvents()` sweep (the DB never - * had them), so an ephemeral with an `expiration` tag will linger - * until it's superseded or until the projection is closed. - * - * **Lifecycle**: this class is a pure state machine with no - * coroutine ownership. Construct it, call [seed] once, then call - * [apply] for each [StoreChange] from [ObservableEventStore.changes]. - * For the standard "wrap into a Flow and collect from a ViewModel" - * use case, use the [ObservableEventStore.project] extension which - * does this composition for you. + * Ephemeral events appear in the snapshot for as long as the + * projection is alive; they never survive a re-seed (the inner store + * never had them) and aren't touched by `deleteExpiredEvents()`. */ class EventStoreProjection( private val store: ObservableEventStore, @@ -174,9 +139,8 @@ class EventStoreProjection( dropWhere { ev -> storeEvent.filters.any { it.match(ev) } } } - // Store's sweep uses strict `<`; isExpirationBefore is - // `<=`, so subtract 1 to match. is StoreChange.DeleteExpired -> { + // Store's sweep uses strict `<`; isExpirationBefore is `<=`, so subtract 1. val cutoff = (storeEvent.asOf ?: nowProvider()) - 1 dropWhere { it.isExpirationBefore(cutoff) } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt index a65c1e5e1..1def6bf4f 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt @@ -32,31 +32,18 @@ import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.asSharedFlow /** - * A reactive façade over any [IEventStore] that publishes every event - * accepted for *observation* — a superset of the events the inner - * store persists. + * Reactive façade over any [IEventStore]. Publishes a [StoreChange] + * on [changes] for every accepted mutation so projections can stay in + * sync without re-querying. * - * The split between persistence and observation is the whole point of - * this class: - * - * - **Non-ephemeral events** are forwarded to the inner store. If the - * inner store rejects (expired, NIP-09 / NIP-62 tombstone, NIP-01 - * supersession loser), the rejection propagates and nothing is - * emitted on [changes]. - * - **Ephemeral events** (kinds `20000-29999`) skip the inner store - * entirely — they're never persisted — but they still emit on - * [changes] so projections can render them while they live. Already - * expired ephemerals are silently dropped. - * - * Wrap any store you want to observe — `SQLiteEventStore`, FS-backed, - * an in-memory test fake — and feed `EventStoreProjection` from the - * resulting [changes] flow. - * - * Reads (`query`, `count`) and out-of-band writes (`delete`, - * `deleteExpiredEvents`) forward to the inner store; the latter are - * also surfaced on [changes] as [StoreChange.DeleteByFilter] / - * [StoreChange.DeleteExpired] so projections can drop the matching - * slots in memory without re-querying. + * - Non-ephemeral [insert] / [transaction] entries forward to the + * inner store. On rejection (expired, NIP-09 / NIP-62 tombstone, + * NIP-01 supersession loser) the throw propagates and nothing is + * emitted. + * - Ephemeral events (kinds `20000-29999`) skip the inner store but + * still emit. Already-expired ephemerals are silently dropped. + * - [delete] and [deleteExpiredEvents] also emit so projections drop + * the matching slots in memory. */ class ObservableEventStore( val inner: IEventStore, @@ -70,35 +57,15 @@ class ObservableEventStore( onBufferOverflow = BufferOverflow.SUSPEND, ) - /** - * Stream of mutations accepted by the observable layer. One - * emission per successful [insert] (or per accepted event in a - * [transaction] body), one emission per [delete] / - * [deleteExpiredEvents] call. Rejected inserts and rolled-back - * transactions emit nothing. - * - * Projections consume this stream — see `EventStoreProjection` - * for how each [StoreChange] is interpreted. - */ + /** Stream of mutations accepted by this layer. See [StoreChange] for the cases. */ val changes: SharedFlow = _changes.asSharedFlow() /** - * Mutations published by [changes]. Projections react to these to - * keep their in-memory view in sync with the underlying store. - * - * - [Insert] is emitted for every event accepted by the - * observable layer (persistable or ephemeral). Carries the - * event itself so the projection can run its NIP-01 / NIP-09 / - * NIP-62 interpretation. - * - [DeleteByFilter] is emitted for every `delete(filter)` / - * `delete(filters)` call on the observable. Carries the same - * filters the store used so projections can apply - * [Filter.match] in memory and drop the matching slots without - * re-querying. - * - [DeleteExpired] is emitted for every `deleteExpiredEvents()` - * sweep. The optional [DeleteExpired.asOf] cutoff lets the - * store pin the timestamp it actually used, so the projection - * drops exactly the events the store dropped. + * One [Insert] per accepted event; one [DeleteByFilter] per + * `delete(filter[s])`; one [DeleteExpired] per + * `deleteExpiredEvents()` sweep. The cutoff carried by + * [DeleteExpired] is pinned at the moment the sweep ran so + * projections drop exactly the events the store dropped. */ sealed interface StoreChange { data class Insert( From 058c56dc215f69a3770c86c66c78247e6a736554 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 16:10:47 +0000 Subject: [PATCH 22/24] feat(quartz): re-evaluate filter membership on addressable supersession MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a replaceable / addressable update arrives in handleInsert, the projection now re-runs Filter.match against the new event for every filter. A v2 that no longer matches a filter (e.g. tag list changed) is removed from that filter's set; a v2 that newly matches a filter joins it. If no filter retains the slot afterwards, it's fully dropped from byId / byAddress. Closes the previous gap where v2 of an addressable kept a stale filter membership inherited from v1. The slot's MutableStateFlow is still updated in place — collectors of that handle still see the content change before the membership update emits. The cap-eviction-with-cleanup logic was extracted into a small `admit` helper since the supersession path and the new-slot path both need it. For the common case (filter on `kinds + authors`, no tag/time constraints), v2 always still matches — the new branches are no-ops and the list reference stays stable, preserving the in-place update guarantee. The behaviour change kicks in for tag, time-window, or id-list filters. New test addressableUpdateDropsSlotWhenFilterStopsMatching: v1 has hashtag "nostr" and matches a `t = nostr` filter; v2 changes to "bitcoin"; the projection drops the slot. 18/18 projection tests pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../cache/projection/EventStoreProjection.kt | 89 +++++++++++++------ .../projection/EventStoreProjectionTest.kt | 44 +++++++++ 2 files changed, 104 insertions(+), 29 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt index 31092bd34..d734e7e95 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt @@ -71,12 +71,13 @@ sealed interface ProjectionState { * list. Stable list reference while membership is unchanged. * - **In-place replaceable / addressable update**: same slot, new * [MutableStateFlow.value]. Only collectors of that handle - * re-render. The slot's sort key is frozen at insertion, so the - * list does not reshuffle when the new version has a later - * `created_at`. *Caveat: the slot stays in whatever filters - * matched the original version; if the new version no longer - * matches a filter (e.g. its tags changed), the slot is not - * re-evaluated and remains live.* + * re-render. The slot's sort key is frozen at insertion so the + * list ordering doesn't reshuffle on a later `created_at`. Filter + * membership *is* re-evaluated against the new event, so a v2 + * that no longer matches a filter (e.g. tag changed) drops out; + * a v2 that newly matches another filter joins it. In the common + * case (filter on `kinds + authors`) v2 still matches and the + * list reference stays stable. * - **Removal**: NIP-09, NIP-62, NIP-40 expiration, `delete(filter)`. * * Limit is **per-filter**: each filter retains at most its own @@ -194,17 +195,37 @@ class EventStoreProjection( if (existing != null) { if (!supersedes(event, existing.flow.value)) return false - // Same address, new winner. Rekey byId from the - // previous event id to the new one and update the - // handle's value in place — list reference stays the - // same; only the handle's collectors re-render. + // Same address, new winner. Rekey byId, mutate + // flow.value in place, then re-evaluate filter + // membership against the new event — v2 may add + // matches or lose previously-matching filters. val previousId = existing.flow.value.id if (previousId != event.id) { byId.remove(previousId) byId[event.id] = existing } existing.flow.value = event as T - return false + + var changed = false + for ((f, set) in perFilter) { + val nowMatches = f.match(event) + val wasIn = set.contains(existing) + when { + nowMatches && !wasIn -> { + if (admit(existing, f, set)) changed = true + } + + !nowMatches && wasIn -> { + set.remove(existing) + changed = true + } + } + } + // If no filter retains the slot anymore, fully drop it. + if (perFilter.values.none { it.contains(existing) }) { + if (removeIndexes(existing)) changed = true + } + return changed } } else if (byId.containsKey(event.id)) { return false @@ -213,32 +234,42 @@ class EventStoreProjection( // Genuinely new slot. Offer it to every matching filter; if // any filter still holds it after cap-eviction, the slot // becomes live and gets indexed. - var membershipChanged = false + var changed = false val slot = Slot(event as T) for ((f, set) in perFilter) { if (!f.match(event)) continue - set.add(slot) - val cap = f.limit ?: continue - while (set.size > cap) { - val tail = set.last() - set.remove(tail) - if (tail !== slot && perFilter.values.none { it.contains(tail) }) { - // Tail no longer retained by any filter — fully drop. - if (removeIndexes(tail)) membershipChanged = true - } - } + if (admit(slot, f, set)) changed = true } - - // The slot survived cap-eviction in at least one filter, so it - // belongs in the indexes. Otherwise nothing was indexed and - // the only membership effect is whatever evictions happened - // along the way. if (perFilter.values.any { it.contains(slot) }) { byId[event.id] = slot if (address != null) byAddress[address] = slot - membershipChanged = true + changed = true } - return membershipChanged + return changed + } + + /** + * Add [slot] to filter [f]'s [set] and evict the tail if the cap + * is exceeded. Returns true if an eviction triggered a slot drop + * from the indexes (its only filter retention was the evicted + * one). + */ + private fun admit( + slot: Slot, + f: Filter, + set: java.util.SortedSet>, + ): Boolean { + set.add(slot) + val cap = f.limit ?: return false + var changed = false + while (set.size > cap) { + val tail = set.last() + set.remove(tail) + if (tail !== slot && perFilter.values.none { it.contains(tail) }) { + if (removeIndexes(tail)) changed = true + } + } + return changed } private fun handleDeletion(deletion: DeletionEvent): Boolean { diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt index 68a98e28b..992b43620 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjectionTest.kt @@ -27,6 +27,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.nip01Core.tags.hashtags.hashtag import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent @@ -208,6 +209,49 @@ class EventStoreProjectionTest { assertSame(seedList, projection.items, "addressable update must not change list reference") } + /** + * If v2 of an addressable no longer matches the filter (e.g. + * tag list changed), the slot is dropped. The projection + * re-evaluates filter membership on every supersession. + */ + @Test + fun addressableUpdateDropsSlotWhenFilterStopsMatching() = + runBlocking { + val time = TimeUtils.now() + // v1 carries tag "nostr"; v2 changes the tag to "bitcoin". + val v1 = + signer.sign( + LongTextNoteEvent.build("blog v1", "title", dTag = "blog", createdAt = time) { + hashtag("nostr") + }, + ) + observable.insert(v1) + + // Filter narrows to events that ALSO carry hashtag "nostr". + val projection = + projectionOf( + Filter( + kinds = listOf(LongTextNoteEvent.KIND), + authors = listOf(v1.pubKey), + tags = mapOf("t" to listOf("nostr")), + ), + ) + projection.awaitLoaded() + assertEquals(1, projection.items.size) + + val v2 = + signer.sign( + LongTextNoteEvent.build("blog v2", "title", dTag = "blog", createdAt = time + 1) { + hashtag("bitcoin") + }, + ) + observable.insert(v2) + + // v2 doesn't match — slot drops and the snapshot is empty. + val after = projection.awaitItems { it.isEmpty() } + assertTrue(after.isEmpty()) + } + /** * Out-of-order arrival: the projection sees the *newer* version * first (e.g. the relay sent v2 first), then v1. The projection From 4cfd56da3681a33a44bb48e82d49a38972e9f900 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 16:24:15 +0000 Subject: [PATCH 23/24] perf(quartz): cheaper hot paths in EventStoreProjection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small wins on paths the projection runs every event arrival or every snapshot publish: 1. Cache `address` on Slot. Previously `removeIndexes(slot)` called `addressOf(slot.flow.value)` on every drop — for replaceables that's a fresh Address allocation every time. Now the address is computed once at slot construction and reused on removal. 2. Single-filter snapshot fast path. The deduped sorted union via TreeSet was unnecessary when there's only one per-filter set — that set is already sorted in the slot comparator's order. The common UI case (one filter per projection) now skips the TreeSet allocation + log-n inserts entirely. 3. Comparator singleton. `slotComparator()` was allocating a fresh Comparator lambda on every call (ctor + every snapshot). Replaced with a single `Comparator>` cast at the use site — the comparator only reads sort keys that don't depend on the type parameter. Also trimmed a verbose comment block in `project()` (3 lines → 1). All 18 projection tests + 240 store + interner tests still pass. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- .../cache/projection/EventStoreProjection.kt | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt index d734e7e95..ba8fd803b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt @@ -148,13 +148,16 @@ class EventStoreProjection( } /** - * Current snapshot as a [ProjectionState.Loaded]. Cheap — the - * deduped sorted union over [perFilter] sets is computed on - * demand. Returns an empty [ProjectionState.Loaded] when no slots - * are live. + * Current snapshot as a [ProjectionState.Loaded]. The + * single-filter case skips the union pass — that filter's set + * is already sorted. Multi-filter projections dedup + merge + * via a transient TreeSet. */ fun snapshot(): ProjectionState.Loaded { if (byId.isEmpty()) return ProjectionState.Loaded(emptyList()) + if (perFilter.size == 1) { + return ProjectionState.Loaded(perFilter.values.first().map { it.flow }) + } val union = sortedSetOf(slotComparator()) for (set in perFilter.values) union.addAll(set) return ProjectionState.Loaded(union.map { it.flow }) @@ -314,20 +317,19 @@ class EventStoreProjection( /** * Remove a slot from [byId] / [byAddress] without touching the * per-filter sets. Used by the per-filter eviction loop, which - * already owns the bookkeeping for those. + * already owns that bookkeeping. */ private fun removeIndexes(slot: Slot): Boolean { val removed = byId.remove(slot.flow.value.id) != null if (!removed) return false - addressOf(slot.flow.value)?.let(byAddress::remove) + slot.address?.let(byAddress::remove) return true } /** - * Internal slot. Each event added to the projection lives inside - * one of these for as long as it survives. The sort key is frozen - * at construction time — supersession in-place updates rewrite - * `flow.value` but never the sort key, so the position inside + * Wraps an event with the indexes the projection needs. Sort key + * and address are frozen at construction — in-place supersession + * updates rewrite `flow.value` but the slot's position inside * each [perFilter] set is stable across updates. */ private class Slot( @@ -335,6 +337,7 @@ class EventStoreProjection( ) { val sortCreatedAt: Long = initial.createdAt val sortId: HexKey = initial.id + val address: Address? = addressOf(initial) val flow: MutableStateFlow = MutableStateFlow(initial) } @@ -370,18 +373,25 @@ class EventStoreProjection( private fun ownerOf(event: Event): HexKey = (event as? GiftWrapEvent)?.recipientPubKey() ?: event.pubKey /** - * created_at DESC, id ASC. The keys are snapshots taken at - * insertion time, so a slot's position never changes after it + * created_at DESC, id ASC. Sort keys are frozen at slot + * construction, so a slot's position never changes after it * joins a set. Distinct events have distinct ids, so no third * tiebreak is needed. + * + * Singleton — stored as `Comparator>` and cast at the + * use site, since the comparator only reads `sortCreatedAt` + * and `sortId` which don't depend on `T`. */ - private fun slotComparator(): Comparator> = + private val SLOT_COMPARATOR: Comparator> = Comparator { a, b -> if (a === b) return@Comparator 0 val byTime = b.sortCreatedAt.compareTo(a.sortCreatedAt) if (byTime != 0) return@Comparator byTime a.sortId.compareTo(b.sortId) } + + @Suppress("UNCHECKED_CAST") + private fun slotComparator(): Comparator> = SLOT_COMPARATOR as Comparator> } } @@ -407,17 +417,13 @@ class EventStoreProjection( fun ObservableEventStore.project(filters: List): Flow> = flow { val projection = EventStoreProjection(this@project, filters) - // Capture the outer collector so we can emit ProjectionState - // from inside `changes.onSubscription { }` and `collect { }`, - // where the implicit `this` is FlowCollector. + // `onSubscription` runs after the SharedFlow subscription is + // active but before events are pulled, so emissions during + // seed land in the buffer and we drain them via `apply` when + // collect proceeds. `outer` bridges the StoreChange-typed + // collectors back to our ProjectionState output. val outer = this emit(ProjectionState.Loading) - // `onSubscription` runs after the SharedFlow subscription is - // active but before we pull events — the buffer absorbs - // emissions arriving during the seed query and we drain them - // through `apply` once collect proceeds. Doing the seed - // inside `collect { }` instead would race with concurrent - // inserts. changes .onSubscription { projection.seed() From c873385fd6aa59062027e5b2e7fdaf4c43161d87 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 17:28:17 +0000 Subject: [PATCH 24/24] docs(quartz): add module-level README with layer overview + tutorials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tutorial-style README covering the full pipeline: relay → NostrClient → ObservableEventStore → InterningEventStore → EventStore → project() → ViewModel → Compose. Two worked examples: 1. Wiring NostrClient subscriptions into the store via SubscriptionListener.onEvent. 2. Building a reactive feed UI with project().stateIn(...) and Compose, showing how the three reactivity layers (Loading/Loaded state, list membership, per-event handles) map to Compose's recomposition model. Cross-links the existing CLIENT.md, RELAY.md, and store/sqlite README. Pointers to per-class KDoc for projection internals. https://claude.ai/code/session_01Jny85MTu1ynKgFBgysfWu5 --- quartz/README.md | 143 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 quartz/README.md diff --git a/quartz/README.md b/quartz/README.md new file mode 100644 index 000000000..f9732c119 --- /dev/null +++ b/quartz/README.md @@ -0,0 +1,143 @@ +# Quartz + +A Kotlin Multiplatform Nostr library — protocol, signing, relay client, event store, and reactive projections. No UI, no Android dependencies in `commonMain`. Targets Android, JVM/Desktop, iOS, and Linux. + +## Layered architecture + +Build apps by composing layers from durable storage at the bottom up to UI projections at the top: + +``` +┌──────────────────────────────────────────┐ +│ UI / ViewModel │ +│ observable.project(filter) │ +│ .stateIn(scope, ...) │ ← reactive list of MutableStateFlow +└──────────────────────────────────────────┘ + ▲ +┌──────────────────────────────────────────┐ +│ ObservableEventStore │ +│ publishes StoreChange on `changes` │ ← bus for reactive consumers +└──────────────────────────────────────────┘ + ▲ +┌──────────────────────────────────────────┐ +│ InterningEventStore │ +│ one Event instance per id, weak refs │ ← shared identity across reads +└──────────────────────────────────────────┘ + ▲ +┌──────────────────────────────────────────┐ +│ EventStore (SQLite) / FsEventStore │ +│ durable, NIP-01/09/40/62 enforced │ ← persistence + Nostr semantics +└──────────────────────────────────────────┘ + ▲ +┌──────────────────────────────────────────┐ +│ NostrClient │ +│ relay subscriptions, NIP-01 messages │ ← network +└──────────────────────────────────────────┘ +``` + +Each layer is optional — you can use just `NostrClient` (see [CLIENT.md](CLIENT.md)), just `EventStore` (see [the SQLite store README](src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/README.md)), or compose the full pipeline below. + +## Wiring relays into the store + +`NostrClient` delivers events through a `SubscriptionListener.onEvent` callback. Pipe each event into the `ObservableEventStore`, which persists it (via the inner store) and publishes it to every open projection: + +```kotlin +// Application init — one set of these per process. +val sqlite = EventStore(dbName = "events.db", relay = "wss://relay.damus.io".normalizeRelayUrl()) +val observable = ObservableEventStore(InterningEventStore(sqlite)) + +val client = NostrClient(websocketBuilder = ktorBuilder) +client.connect() + +// Open a relay subscription that pumps every arriving event into the store. +client.subscribe( + subId = "home", + filters = mapOf( + "wss://relay.damus.io".normalizeRelayUrl() to listOf( + Filter(kinds = listOf(1), authors = followedAuthors, limit = 500), + ), + ), + listener = object : SubscriptionListener { + override fun onEvent(event: Event, isLive: Boolean, relay: NormalizedRelayUrl, forFilters: List?) { + applicationScope.launch { observable.insert(event) } + } + }, +) + +// Periodic NIP-40 sweep — projections drop expired events when this fires. +applicationScope.launch { + while (isActive) { delay(15.minutes); observable.deleteExpiredEvents() } +} +``` + +`observable.insert(event)` is idempotent under NIP-01 supersession (older replaceables / addressables are rejected by the inner store) and validates expiration / vanish tombstones, so it's safe to fire-and-forget for every relay arrival. + +`InterningEventStore` keeps one `Event` instance alive per id (weakly, via `EventInterner.Default`), so events that re-arrive from multiple relays — or get re-read by query — share the same object reference as long as some projection holds them. + +## Building a reactive feed UI + +A feed screen reads from the store via `ObservableEventStore.project()`, which returns a cold `Flow>`. Wrap it with `stateIn(...)` in a ViewModel and collect from Compose: + +```kotlin +class HomeFeedViewModel( + observable: ObservableEventStore, + followedAuthors: List, +) : ViewModel() { + val state: StateFlow> = + observable + .project(Filter(kinds = listOf(1), authors = followedAuthors, limit = 200)) + .stateIn(viewModelScope, SharingStarted.WhileSubscribed(5_000), ProjectionState.Loading) +} + +@Composable +fun HomeFeedScreen(vm: HomeFeedViewModel) { + val state by vm.state.collectAsState() + when (val s = state) { + is ProjectionState.Loading -> SpinnerScaffold() + is ProjectionState.Loaded -> { + // Layer 1: list reference is stable while membership is unchanged. + // LazyColumn only invalidates structure on inserts / removals. + LazyColumn { + items(s.items, key = { it.value.id }) { handle -> + NoteRow(handle) + } + } + } + } +} + +@Composable +fun NoteRow(handle: MutableStateFlow) { + // Layer 2: only collectors of THIS handle re-render when the + // event mutates in place (addressable supersession, etc.). + val event by handle.collectAsState() + Column { + Text(event.content) + // Layer 3: derived flows — e.g. counters reactive to OTHER projections. + ReactionRow(event.id) + } +} +``` + +The three layers map cleanly to Compose's recomposition model: + +- `state` re-renders the screen scaffolding (`Loading` vs `Loaded`). +- `s.items` re-emits only when membership changes (insert, NIP-09 deletion, NIP-62 vanish, NIP-40 expiration sweep, manual `delete(filter)`). +- `handle` re-emits only when its specific event mutates in place (e.g. a new version of a kind-30023 long-form post supersedes the previous one). + +## Publishing from the UI + +```kotlin +val signer = NostrSignerInternal(KeyPair()) +val signed = signer.sign(TextNoteEvent.build("hello nostr", createdAt = TimeUtils.now())) +observable.insert(signed) // hits the bus → all open projections see it +client.send(signed, relays = ...) // also publish to relays +``` + +The projection runs NIP-01 supersession, NIP-09 author checks, NIP-62 vanish scoping, and NIP-40 expiration filtering automatically. UI code never needs to know any of those rules. + +## Where to read more + +- [`CLIENT.md`](CLIENT.md) — building a relay client with `NostrClient` and Ktor. +- [`RELAY.md`](RELAY.md) — running a relay server with Quartz. +- [SQLite event store README](src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/README.md) — query planner, indexing strategies, NIP-09/40/62 enforcement details. +- KDoc on `EventStoreProjection`, `ObservableEventStore`, `EventInterner` — package-level reference for the projection layer.