nick/amethyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(quartz/sqlite): NIP compliance and migration safety in event store

Browse Source 
- NIP-09 (#2): a-tag and replaceable deletes now respect
  `created_at <= deletion.created_at` so a stale deletion request
  cannot remove a newer addressable that legitimately replaced it.
  `+created_at` hint on the addressable path keeps the d_tag-selective
  index in use.

- NIP-01 (#1): replaceable / addressable triggers now apply the
  lexical-id tiebreaker — when two events share `created_at`, the
  one with the lexicographically smaller id wins, matching the spec.

- Schema (#5): FullTextSearchModule.versionFinder probes FTS support
  with a dummy table, but used to leave it behind. The first v1->v2
  upgrade then failed because re-running create() would hit
  "already exists". Now we drop the probe table immediately and
  defensively clean up any stragglers.

- Schema (#6): onCreate / onUpgrade and the matching `setUserVersion`
  are now wrapped in a single transaction so a partial migration
  cannot leave the DB with mismatched user_version and schema.

- SQL DSL (#4): Condition.NotEquals(null) now produces `IS NOT NULL`
  instead of `IS NULL`.

- Doc fix (#13): swapped vacuum/analyse comments now describe the
  right command.

Tests: same-`created_at` tiebreaker for replaceables and addressables,
NIP-09 created_at window for a-tag deletes, schema drop+recreate
idempotency (covers the FTS dummy-table regression).

https://claude.ai/code/session_01X163Nr31vGkvAXoJ3JgMov
This commit is contained in:
Claude
2026-04-25 12:32:11 +00:00
parent fb478d6019
commit 28b23b5e63
10 changed files with 265 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableModule.kt
+13 -8
View File
@@ -32,12 +32,14 @@ class AddressableModule : IModule {
""".trimIndent(),
)
// deletes older addressables when inserting new ones
// if a newer addressable is inserted the unique index
// above will be triggered. Delete cascade will take
// care of the event_tags table
// the duplicate: kind >= 30000 AND kind < 40000
// helps SQLlite find the index above
// deletes older addressables when inserting new ones.
// Per NIP-01, the "older" version of an addressable is the one
// with the smaller created_at, OR with equal created_at and
// a lexicographically larger id (lowest id wins).
// If a newer addressable is inserted the unique index above
// will be triggered. Delete cascade will take care of the
// event_tags table. The duplicate `kind >= 30000 AND kind < 40000`
// clause helps SQLite pick the partial unique index above.
db.execSQL(
"""
CREATE TRIGGER delete_older_addressable_event
@@ -50,8 +52,11 @@ class AddressableModule : IModule {
event_headers.kind = NEW.kind AND
event_headers.pubkey = NEW.pubkey AND
event_headers.d_tag = NEW.d_tag AND
event_headers.created_at < NEW.created_at AND
event_headers.kind >= 30000 AND event_headers.kind < 40000;
event_headers.kind >= 30000 AND event_headers.kind < 40000 AND
(
event_headers.created_at < NEW.created_at OR
(event_headers.created_at = NEW.created_at AND event_headers.id > NEW.id)
);
END;
""".trimIndent(),
)
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt
+12 -6
View File
@@ -87,7 +87,7 @@ class DeletionRequestModule(
val idValues = event.deleteEventIds()
val addresses = event.deleteAddresses()
deleteSQL(event.pubKey, idValues, addresses, hasher(db)).forEach { delete ->
deleteSQL(event.pubKey, event.createdAt, idValues, addresses, hasher(db)).forEach { delete ->
db.execSQL(delete.sql, delete.args)
}
}
@@ -97,10 +97,14 @@ class DeletionRequestModule(
* Creates a Delete statement that correctly deletes by id,
* by address and by replaceable (no d-tag) using each index
* appropriately, including GiftWraps where the owner is the
* p-tag (via event_header.pubkey_owner_hash)
* p-tag (via event_header.pubkey_owner_hash).
*
* Per NIP-09, address-based deletes (a-tag) MUST only remove
* events with `created_at <= deletion.created_at`.
*/
fun deleteSQL(
pubkey: HexKey,
createdAt: Long,
idValues: List<String>,
addresses: List<Address>,
hasher: com.vitorpamplona.quartz.nip01Core.store.sqlite.TagNameValueHasher,
@@ -156,9 +160,10 @@ class DeletionRequestModule(
WHERE (
$addressableParams
) AND
kind >= 30000 AND kind < 40000
kind >= 30000 AND kind < 40000 AND
+created_at <= ?
""".trimIndent(),
addressableValues.toTypedArray(),
(addressableValues + createdAt).toTypedArray(),
)
} else {
null
@@ -172,9 +177,10 @@ class DeletionRequestModule(
WHERE
kind IN ($replaceableKindsParam) AND
pubkey = ? AND
((kind in (0,3)) OR (kind >= 10000 AND kind < 20000))
((kind in (0,3)) OR (kind >= 10000 AND kind < 20000)) AND
created_at <= ?
""".trimIndent(),
replaceableKindsValues.plus(pubkey).toTypedArray(),
(replaceableKindsValues.plus(pubkey) + createdAt).toTypedArray(),
)
} else {
null
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/FullTextSearchModule.kt
+26 -11
View File
@@ -31,7 +31,7 @@ class FullTextSearchModule : IModule {
val contentName = "content"
override fun create(db: SQLiteConnection) {
val ftsVersion = FullTextSearchModule().versionFinder(db)
val ftsVersion = versionFinder(db)
db.execSQL(
"""
CREATE VIRTUAL TABLE $tableName
@@ -77,19 +77,34 @@ class FullTextSearchModule : IModule {
}
}
fun versionFinder(db: SQLiteConnection): Int =
try {
fun versionFinder(db: SQLiteConnection): Int {
// Defensive cleanup in case a previous probe left these behind
// (e.g. a partial create() during an upgrade) — without this,
// the next CREATE VIRTUAL TABLE would fail with "already exists".
db.execSQL("DROP TABLE IF EXISTS dummy_fts5")
db.execSQL("DROP TABLE IF EXISTS dummy_fts4")
db.execSQL("DROP TABLE IF EXISTS dummy_fts3")
val (table, version) =
try {
db.execSQL("CREATE VIRTUAL TABLE dummy_fts5 USING fts5(dummy)")
5
try {
db.execSQL("CREATE VIRTUAL TABLE dummy_fts5 USING fts5(dummy)")
"dummy_fts5" to 5
} catch (e: SQLiteException) {
db.execSQL("CREATE VIRTUAL TABLE dummy_fts4 USING fts4(dummy)")
"dummy_fts4" to 4
}
} catch (e: SQLiteException) {
db.execSQL("CREATE VIRTUAL TABLE dummy_fts4 USING fts4(dummy)")
4
db.execSQL("CREATE VIRTUAL TABLE dummy_fts3 USING fts3(dummy)")
"dummy_fts3" to 3
}
} catch (e: SQLiteException) {
db.execSQL("CREATE VIRTUAL TABLE dummy_fts3 USING fts3(dummy)")
3
}
// We only needed the table to probe FTS support — drop it now so
// re-running create() on an already-probed DB stays idempotent.
db.execSQL("DROP TABLE $table")
return version
}
override fun deleteAll(db: SQLiteConnection) {
db.execSQL("DELETE FROM event_fts")
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableModule.kt
+11 -6
View File
@@ -32,10 +32,13 @@ class ReplaceableModule : IModule {
""".trimIndent(),
)
// deletes older addressables when inserting new ones
// if a newer addressable is inserted the unique index
// above will be triggered. Delete cascade will take
// care of the event_tags table
// deletes older replaceables when inserting new ones.
// Per NIP-01, the "older" version of a replaceable is the one
// with the smaller created_at, OR with equal created_at and
// a lexicographically larger id (lowest id wins).
// If a newer replaceable is inserted the unique index above
// will be triggered. Delete cascade will take care of the
// event_tags table.
db.execSQL(
"""
CREATE TRIGGER delete_older_replaceable_event
@@ -43,12 +46,14 @@ class ReplaceableModule : IModule {
FOR EACH ROW
WHEN (NEW.kind IN (0, 3)) OR (NEW.kind >= 10000 AND NEW.kind < 20000)
BEGIN
-- Delete older records if this is the newest
DELETE FROM event_headers
WHERE
event_headers.kind = NEW.kind AND
event_headers.pubkey = NEW.pubkey AND
event_headers.created_at < NEW.created_at;
(
event_headers.created_at < NEW.created_at OR
(event_headers.created_at = NEW.created_at AND event_headers.id > NEW.id)
);
END;
""".trimIndent(),
)
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt
+12 -8
View File
@@ -108,11 +108,15 @@ class SQLiteEventStore(
val currentVersion = getUserVersion(db)
if (currentVersion == 0) {
onCreate(db)
setUserVersion(db, DATABASE_VERSION)
db.transaction {
onCreate(this)
setUserVersion(this, DATABASE_VERSION)
}
} else if (currentVersion < DATABASE_VERSION) {
onUpgrade(db, currentVersion, DATABASE_VERSION)
setUserVersion(db, DATABASE_VERSION)
db.transaction {
onUpgrade(this, currentVersion, DATABASE_VERSION)
setUserVersion(this, DATABASE_VERSION)
}
}
return db
@@ -160,16 +164,16 @@ class SQLiteEventStore(
}
suspend fun vacuum() {
// 1. ANALYZE: Collects statistics about tables and indices
// to help the query planner optimize queries.
// VACUUM: Rebuilds the database file, reclaiming unused space
// and reducing fragmentation.
withContext(Dispatchers.IO) {
connection.execSQL("VACUUM")
}
}
suspend fun analyse() {
// 2. VACUUM: Rebuilds the database file, reclaiming unused space
// and reducing fragmentation.
// ANALYZE: Collects statistics about tables and indices
// to help the query planner optimize queries.
withContext(Dispatchers.IO) {
connection.execSQL("ANALYZE")
}
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/sql/SqlSelectionBuilder.kt
+1 -1
View File
@@ -60,7 +60,7 @@ class SqlSelectionBuilder(
is Condition.NotEquals -> {
if (cond.value == null) {
"${cond.column} IS NULL"
"${cond.column} IS NOT NULL"
} else {
selectionArgs.add(cond.value.toString())
"${cond.column} != ?"
quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableTest.kt
+66
View File
@@ -89,6 +89,72 @@ class AddressableTest : BaseDBTest() {
db.assertQuery(null, Filter(ids = listOf(version1.id)))
}
@Test
fun testReplacingSameCreatedAtLowerIdWins() =
forEachDB { db ->
val time = TimeUtils.now()
// Two events with the same (kind, pubkey, d_tag, created_at) but
// different content -> different ids. NIP-01 says the lower-id wins.
val a =
signer.sign(
LongTextNoteEvent.build(
"version A",
"title",
dTag = "tie",
createdAt = time,
),
)
val b =
signer.sign(
LongTextNoteEvent.build(
"version B",
"title",
dTag = "tie",
createdAt = time,
),
)
val (winner, loser) = if (a.id < b.id) a to b else b to a
db.insert(loser)
db.assertQuery(loser, Filter(ids = listOf(loser.id)))
db.insert(winner)
db.assertQuery(null, Filter(ids = listOf(loser.id)))
db.assertQuery(winner, Filter(ids = listOf(winner.id)))
}
@Test
fun testReplacingSameCreatedAtHigherIdRejected() =
forEachDB { db ->
val time = TimeUtils.now()
val a =
signer.sign(
LongTextNoteEvent.build(
"version A",
"title",
dTag = "tie",
createdAt = time,
),
)
val b =
signer.sign(
LongTextNoteEvent.build(
"version B",
"title",
dTag = "tie",
createdAt = time,
),
)
val (winner, loser) = if (a.id < b.id) a to b else b to a
db.insert(winner)
assertFailsWith<SQLiteException> {
db.insert(loser)
}
db.assertQuery(winner, Filter(ids = listOf(winner.id)))
db.assertQuery(null, Filter(ids = listOf(loser.id)))
}
@Test
fun testTriggersIndexUsage() =
forEachDB { db ->
quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BasicTest.kt
+19
View File
@@ -200,6 +200,25 @@ class BasicTest : BaseDBTest() {
}
}
@Test
fun testSchemaRecreateIsIdempotent() =
forEachDB { db ->
// The v1->v2 upgrade in SQLiteEventStore.onUpgrade does
// modules.reversed().forEach { it.drop(db) } then
// modules.forEach { it.create(db) }. Pre-fix, FullTextSearchModule
// left dummy_fts3/4/5 tables behind on first probe, so the
// second create() would throw "already exists".
db.store.modules
.reversed()
.forEach { it.drop(db.store.connection) }
db.store.modules.forEach { it.create(db.store.connection) }
// After re-creation the store is still usable.
val note = signer.sign(TextNoteEvent.build("test1"))
db.store.insertEvent(note)
db.store.assertQuery(note, Filter(ids = listOf(note.id)))
}
@Test
fun hashCodeTest() =
forEachDB { db ->
quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt
+68 -11
View File
@@ -93,7 +93,9 @@ class DeletionTest : BaseDBTest() {
db.assertQuery(null, Filter(ids = listOf(note2.id)))
db.assertQuery(note3, Filter(ids = listOf(note3.id)))
val deletion = signer.sign(DeletionEvent.build(listOf(note1)))
// Deletion must be timestamped >= note3.createdAt for it to
// remove the latest addressable per NIP-09.
val deletion = signer.sign(DeletionEvent.build(listOf(note1), createdAt = time + 100))
db.insert(deletion)
@@ -128,7 +130,7 @@ class DeletionTest : BaseDBTest() {
db.assertQuery(null, Filter(ids = listOf(note2.id)))
db.assertQuery(note3, Filter(ids = listOf(note3.id)))
val deletion = signer.sign(DeletionEvent.buildAddressOnly(listOf(note1)))
val deletion = signer.sign(DeletionEvent.buildAddressOnly(listOf(note1), createdAt = time + 100))
db.insert(deletion)
@@ -226,6 +228,7 @@ class DeletionTest : BaseDBTest() {
db.store.deletionModule
.deleteSQL(
pubkey = "key1",
createdAt = 1766686500,
idValues = listOf("ca29c211f", "ca29c211d"),
addresses = emptyList(),
hasher =
@@ -256,6 +259,7 @@ class DeletionTest : BaseDBTest() {
db.store.deletionModule
.deleteSQL(
pubkey = "key1",
createdAt = 1766686500,
idValues = emptyList(),
addresses =
listOf(
@@ -273,8 +277,9 @@ class DeletionTest : BaseDBTest() {
WHERE (
(kind = "30000" AND pubkey = "key1" AND d_tag = "a")
) AND
kind >= 30000 AND kind < 40000
├── SEARCH event_headers USING COVERING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?)
kind >= 30000 AND kind < 40000 AND
+created_at <= "1766686500"
├── SEARCH event_headers USING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?)
├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?)
├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?)
└── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?)
@@ -290,6 +295,7 @@ class DeletionTest : BaseDBTest() {
db.store.deletionModule
.deleteSQL(
pubkey = "key1",
createdAt = 1766686500,
idValues = emptyList(),
addresses =
listOf(
@@ -310,8 +316,9 @@ class DeletionTest : BaseDBTest() {
WHERE (
(kind = "30000" AND pubkey = "key1" AND d_tag IN ("a","b","c","d"))
) AND
kind >= 30000 AND kind < 40000
├── SEARCH event_headers USING COVERING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?)
kind >= 30000 AND kind < 40000 AND
+created_at <= "1766686500"
├── SEARCH event_headers USING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?)
├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?)
├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?)
└── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?)
@@ -327,6 +334,7 @@ class DeletionTest : BaseDBTest() {
db.store.deletionModule
.deleteSQL(
pubkey = "key1",
createdAt = 1766686500,
idValues = emptyList(),
addresses =
listOf(
@@ -351,12 +359,13 @@ class DeletionTest : BaseDBTest() {
OR
(kind = "30101" AND pubkey = "key1" AND d_tag IN ("c","d"))
) AND
kind >= 30000 AND kind < 40000
kind >= 30000 AND kind < 40000 AND
+created_at <= "1766686500"
├── MULTI-INDEX OR
│ ├── INDEX 1
│ │ └── SEARCH event_headers USING COVERING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?)
│ │ └── SEARCH event_headers USING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?)
│ └── INDEX 2
│ └── SEARCH event_headers USING COVERING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?)
│ └── SEARCH event_headers USING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?)
├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?)
├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?)
└── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?)
@@ -372,6 +381,7 @@ class DeletionTest : BaseDBTest() {
db.store.deletionModule
.deleteSQL(
pubkey = "key1",
createdAt = 1766686500,
idValues = emptyList(),
addresses =
listOf(
@@ -394,8 +404,9 @@ class DeletionTest : BaseDBTest() {
WHERE
kind IN ("10000","10001") AND
pubkey = "key1" AND
((kind in (0,3)) OR (kind >= 10000 AND kind < 20000))
├── SEARCH event_headers USING COVERING INDEX replaceable_idx (kind=? AND pubkey=?)
((kind in (0,3)) OR (kind >= 10000 AND kind < 20000)) AND
created_at <= "1766686500"
├── SEARCH event_headers USING COVERING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=? AND created_at<?)
├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?)
├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?)
└── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?)
@@ -403,4 +414,50 @@ class DeletionTest : BaseDBTest() {
db.store.explainQuery(sql.sql, sql.args),
)
}
@Test
fun testDeletionDoesNotRemoveNewerAddressable() =
forEachDB { db ->
// Old addressable (will be deleted by deletion request)
val old =
signer.sign(
LongTextNoteEvent.build(
"version 1",
"title",
dTag = "blog-1",
createdAt = 1000,
),
)
db.insert(old)
db.assertQuery(old, Filter(ids = listOf(old.id)))
// Deletion targeting the address with created_at = 1500
val deletion =
signer.sign(
DeletionEvent.buildAddressOnly(
listOf(old),
createdAt = 1500,
),
)
db.insert(deletion)
// The old version is gone
db.assertQuery(null, Filter(ids = listOf(old.id)))
// Now a NEWER version of the same address arrives, with
// created_at AFTER the deletion. NIP-09 says it must be kept.
val newer =
signer.sign(
LongTextNoteEvent.build(
"version 2",
"title",
dTag = "blog-1",
createdAt = 2000,
),
)
db.insert(newer)
db.assertQuery(newer, Filter(ids = listOf(newer.id)))
}
}
quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableTest.kt
+37
View File
@@ -89,6 +89,43 @@ class ReplaceableTest : BaseDBTest() {
db.assertQuery(null, Filter(ids = listOf(version1.id)))
}
@Test
fun testReplacingSameCreatedAtLowerIdWins() =
forEachDB { db ->
val time = TimeUtils.now()
// Two events with the same (kind, pubkey, created_at) but different
// content -> different ids. NIP-01 says the lower-id wins.
val a = signer.sign(MetadataEvent.createNew("Vitor A", createdAt = time))
val b = signer.sign(MetadataEvent.createNew("Vitor B", createdAt = time))
val (winner, loser) = if (a.id < b.id) a to b else b to a
// Insert the higher-id (loser) first, then the lower-id (winner).
// The trigger should delete the loser and let the winner in.
db.insert(loser)
db.assertQuery(loser, Filter(ids = listOf(loser.id)))
db.insert(winner)
db.assertQuery(null, Filter(ids = listOf(loser.id)))
db.assertQuery(winner, Filter(ids = listOf(winner.id)))
}
@Test
fun testReplacingSameCreatedAtHigherIdRejected() =
forEachDB { db ->
val time = TimeUtils.now()
val a = signer.sign(MetadataEvent.createNew("Vitor A", createdAt = time))
val b = signer.sign(MetadataEvent.createNew("Vitor B", createdAt = time))
val (winner, loser) = if (a.id < b.id) a to b else b to a
// Winner already in DB; the higher-id duplicate must be rejected.
db.insert(winner)
assertFailsWith<SQLiteException> {
db.insert(loser)
}
db.assertQuery(winner, Filter(ids = listOf(winner.id)))
db.assertQuery(null, Filter(ids = listOf(loser.id)))
}
@Test
fun testTriggersIndexUsageKind0() =
forEachDB { db ->