diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt index 1f7f771b3..bb35f07be 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt @@ -227,8 +227,8 @@ class DeletionTest { ON event_headers.row_id = event_tags.event_header_row_id WHERE event_tags.tag_hash IN (3221122, 223322) AND + event_tags.created_at >= 1766686500 AND event_headers.kind = 5 AND - event_headers.created_at >= 1766686500 AND event_headers.pubkey_owner_hash = 22332323 """.trimIndent() @@ -238,7 +238,7 @@ class DeletionTest { TestCase.assertEquals( """ ${sql.replace("\n","\n ")} - ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=? AND created_at>?) └── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) """.trimIndent(), explainer, diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt index 49c9cf12a..9a6000835 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -31,7 +31,7 @@ import org.junit.Before import org.junit.Test class QueryAssemblerTest { - val hasher = TagNameValueHasher(0L) + val hasher = TagNameValueHasher(0) val builder = EventIndexesModule(FullTextSearchModule(), { hasher }) val key1 = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" @@ -81,13 +81,14 @@ class QueryAssemblerTest { """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_headers.created_at >= "1750889190") AND (event_tags.tag_hash = "2657743813502222172") AND (event_headers.kind = "5") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "2657743813502222172") AND (event_tags.created_at >= "1750889190") AND (event_headers.kind = "5") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY created_at DESC, id ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) - │ └── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=? AND created_at>?) + │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ └── USE TEMP B-TREE FOR DISTINCT ├── SCAN filtered ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY @@ -278,14 +279,14 @@ class QueryAssemblerTest { """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_tags.created_at DESC LIMIT 30 ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY created_at DESC, id ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - │ └── USE TEMP B-TREE FOR ORDER BY + │ └── USE TEMP B-TREE FOR DISTINCT ├── SCAN filtered ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY @@ -309,14 +310,13 @@ class QueryAssemblerTest { """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE event_tags.tag_hash = "-4551135004136952885" ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE event_tags.tag_hash = "-4551135004136952885" ORDER BY event_tags.created_at DESC LIMIT 30 ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY created_at DESC, id ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) - │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - │ └── USE TEMP B-TREE FOR ORDER BY + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) + │ └── USE TEMP B-TREE FOR DISTINCT ├── SCAN filtered ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY @@ -344,14 +344,14 @@ class QueryAssemblerTest { """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_tags.created_at DESC LIMIT 30 ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY created_at DESC, id ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - │ └── USE TEMP B-TREE FOR ORDER BY + │ └── USE TEMP B-TREE FOR DISTINCT ├── SCAN filtered ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY @@ -376,14 +376,14 @@ class QueryAssemblerTest { """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") ORDER BY event_tags.created_at DESC LIMIT 30 ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY created_at DESC, id ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - │ └── USE TEMP B-TREE FOR ORDER BY + │ └── USE TEMP B-TREE FOR DISTINCT ├── SCAN filtered ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY @@ -412,15 +412,15 @@ class QueryAssemblerTest { """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsIn1 ON event_tagsIn1.event_header_row_id = event_tags.event_header_row_id INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tagsIn1.tag_hash = "-6379614208644810021") AND (event_headers.kind = "1") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsIn1 ON event_tagsIn1.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn1.created_at = event_tags.created_at INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tagsIn1.tag_hash = "-6379614208644810021") AND (event_headers.kind = "1") ORDER BY event_tags.created_at DESC LIMIT 30 ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY created_at DESC, id ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - │ ├── SEARCH event_tagsIn1 USING COVERING INDEX query_by_tags_hash (tag_hash=? AND event_header_row_id=?) - │ └── USE TEMP B-TREE FOR ORDER BY + │ ├── SEARCH event_tagsIn1 USING INDEX query_by_tags_hash (tag_hash=? AND event_header_row_id=?) + │ └── USE TEMP B-TREE FOR DISTINCT ├── SCAN filtered ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY @@ -517,13 +517,14 @@ class QueryAssemblerTest { """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsAll0_1 ON event_tagsAll0_1.event_header_row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "884286737453847614") AND (event_tagsAll0_1.tag_hash = "-4988851810256311323") + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsAll0_1 ON event_tagsAll0_1.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll0_1.created_at = event_tags.created_at WHERE (event_tags.tag_hash = "884286737453847614") AND (event_tagsAll0_1.tag_hash = "-4988851810256311323") ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY created_at DESC, id ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) - │ └── SEARCH event_tagsAll0_1 USING COVERING INDEX query_by_tags_hash (tag_hash=? AND event_header_row_id=?) + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_tagsAll0_1 USING INDEX query_by_tags_hash (tag_hash=? AND created_at=?) + │ └── USE TEMP B-TREE FOR DISTINCT ├── SCAN filtered ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) └── USE TEMP B-TREE FOR ORDER BY diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt index 32e567fd1..de7b13e59 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt @@ -43,7 +43,11 @@ class DeletionRequestModule( BEFORE INSERT ON event_headers FOR EACH ROW BEGIN - -- Check for ID-based deletion record + -- Checks if this event hasn't already been deleted. If so, reject it. + + -- Highly optimized using hash + created_at (duplicated in tags) index: + ---- Expects tag_hash to be non-existant for new events (quick exit) + ---- Expects created-at >= event's to not exist (quick exit) SELECT RAISE(ABORT, 'blocked: a deletion event exists') WHERE EXISTS ( SELECT 1 FROM event_tags @@ -51,9 +55,9 @@ class DeletionRequestModule( ON event_headers.row_id = event_tags.event_header_row_id WHERE event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND + event_tags.created_at >= NEW.created_at AND event_headers.kind = 5 AND - event_headers.pubkey_owner_hash = NEW.pubkey_owner_hash AND - event_headers.created_at >= NEW.created_at + event_headers.pubkey_owner_hash = NEW.pubkey_owner_hash ); END; """.trimIndent(), diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt index 9fdd08982..981260c58 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt @@ -63,6 +63,7 @@ class EventIndexesModule( CREATE TABLE event_tags ( event_header_row_id INTEGER NOT NULL, tag_hash INTEGER NOT NULL, + created_at INTEGER NOT NULL, FOREIGN KEY (event_header_row_id) REFERENCES event_headers(row_id) ON DELETE CASCADE ) """.trimIndent(), @@ -70,12 +71,15 @@ class EventIndexesModule( db.execSQL("CREATE UNIQUE INDEX event_headers_id ON event_headers (id)") db.execSQL("CREATE INDEX query_by_kind_pubkey_dtag_idx ON event_headers (kind, pubkey, d_tag)") - db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers (created_at desc, id)") + db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers (created_at DESC, id)") + // need to check if this is actually needed. - db.execSQL("CREATE INDEX query_by_created_at_kind_key ON event_headers (created_at desc, kind, pubkey)") + db.execSQL("CREATE INDEX query_by_created_at_kind_key ON event_headers (created_at DESC, kind, pubkey)") db.execSQL("CREATE INDEX fk_event_tags_header_id ON event_tags (event_header_row_id)") - db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, event_header_row_id)") + + // This is a very slow index to build (half the insert time goes here) but it is extremely effective. + db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, created_at DESC)") // Prevent updates to maintain immutability db.execSQL( @@ -117,9 +121,9 @@ class EventIndexesModule( val sqlInsertTags = """ INSERT OR ROLLBACK INTO event_tags - (event_header_row_id, tag_hash) + (event_header_row_id, tag_hash, created_at) VALUES - (?,?) + (?,?,?) """.trimIndent() fun insert( @@ -177,6 +181,7 @@ class EventIndexesModule( indexableTags.forEach { stmtTags.bindLong(1, headerId) stmtTags.bindLong(2, it) + stmtTags.bindLong(3, event.createdAt) stmtTags.executeInsert() } @@ -462,6 +467,16 @@ class EventIndexesModule( val nonDTagsAll = filter.tagsAll?.filter { it.key != "d" } ?: emptyMap() + val reverseLookup = nonDTagsIn.isNotEmpty() || nonDTagsAll.isNotEmpty() + + val needHeaders = + with(filter) { + (ids != null) || + (authors != null && authors.isNotEmpty()) || + (kinds != null && kinds.isNotEmpty()) || + (tags != null && tags.containsKey("d")) + } + val hasHeaders = with(filter) { (ids != null) || @@ -478,13 +493,13 @@ class EventIndexesModule( val projection = buildString { // always do tags if there are any - if (nonDTagsIn.isNotEmpty() || nonDTagsAll.isNotEmpty()) { + if (reverseLookup) { append("SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags ") // it's quite rare to have 2 tags in the filter, but possible nonDTagsIn.keys.forEachIndexed { index, tagName -> if (defaultTagKey != null) { - append("INNER JOIN event_tags as event_tagsIn$index ON event_tagsIn$index.event_header_row_id = event_tags.event_header_row_id ") + append("INNER JOIN event_tags as event_tagsIn$index ON event_tagsIn$index.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn$index.created_at = event_tags.created_at ") } else { defaultTagKey = TagNameForQuery.InTags(tagName) } @@ -493,14 +508,14 @@ class EventIndexesModule( nonDTagsAll.keys.forEachIndexed { index, tagName -> nonDTagsAll[tagName]!!.forEachIndexed { valueIndex, tagValue -> if (defaultTagKey != null) { - append("INNER JOIN event_tags as event_tagsAll${index}_$valueIndex ON event_tagsAll${index}_$valueIndex.event_header_row_id = event_tags.event_header_row_id ") + append("INNER JOIN event_tags as event_tagsAll${index}_$valueIndex ON event_tagsAll${index}_$valueIndex.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll${index}_$valueIndex.created_at = event_tags.created_at ") } else { defaultTagKey = TagNameForQuery.AllTags(tagName, valueIndex) } } } - if (hasHeaders) { + if (needHeaders) { append("INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id ") } @@ -525,10 +540,6 @@ class EventIndexesModule( // ids reduce the filter the most filter.ids?.let { equalsOrIn("event_headers.id", it) } - // range search is bad but most of the time these are up the top with few elements. - filter.since?.let { greaterThanOrEquals("event_headers.created_at", it) } - filter.until?.let { lessThanOrEquals("event_headers.created_at", it) } - // it's quite rare to have 2 tags in the filter, but possible nonDTagsIn.keys.forEachIndexed { index, tagName -> val column = @@ -560,6 +571,15 @@ class EventIndexesModule( } } + // range search is bad but most of the time these are up the top with few elements. + if (reverseLookup) { + filter.since?.let { greaterThanOrEquals("event_tags.created_at", it) } + filter.until?.let { lessThanOrEquals("event_tags.created_at", it) } + } else { + filter.since?.let { greaterThanOrEquals("event_headers.created_at", it) } + filter.until?.let { lessThanOrEquals("event_headers.created_at", it) } + } + filter.kinds?.let { equalsOrIn("event_headers.kind", it) } filter.authors?.let { equalsOrIn("event_headers.pubkey", it) } @@ -580,7 +600,11 @@ class EventIndexesModule( val whereClause = if (filter.limit != null) { - "${clause.conditions} ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT ${filter.limit}" + if (reverseLookup) { + "${clause.conditions} ORDER BY event_tags.created_at DESC LIMIT ${filter.limit}" + } else { + "${clause.conditions} ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT ${filter.limit}" + } } else { clause.conditions }