diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt index a1fe2259a..6926af8de 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt @@ -26,19 +26,46 @@ import org.junit.After import org.junit.Before open class BaseDBTest { - private lateinit var dbs: Map + private lateinit var dbs: MutableMap + + fun DefaultIndexingStrategy.name(): String = + """ + indexEventsByCreatedAtAlone=$indexEventsByCreatedAtAlone + indexTagsByCreatedAtAlone=$indexTagsByCreatedAtAlone + indexTagsWithKindAndPubkey=$indexTagsWithKindAndPubkey + useAndIndexIdOnOrderBy=$useAndIndexIdOnOrderBy + """.trimIndent() @Before fun setup() { val context = ApplicationProvider.getApplicationContext() - dbs = - mapOf( - "Default" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(false, false)), - "IndexAll" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(true, false)), - "Order by ID" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(false, true)), - "IndexAll, Order by ID" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(true, true)), - ) + val booleans = listOf(true, false) + + dbs = mutableMapOf() + + // tests all possible DBs + for (indexEventsByCreatedAtAlone in booleans) { + for (indexTagsByCreatedAtAlone in booleans) { + for (indexTagsWithKindAndPubkey in booleans) { + for (useAndIndexIdOnOrderBy in booleans) { + val indexStrategy = + DefaultIndexingStrategy( + indexEventsByCreatedAtAlone, + indexTagsByCreatedAtAlone, + indexTagsWithKindAndPubkey, + useAndIndexIdOnOrderBy, + ) + dbs[indexStrategy.name()] = + EventStore( + context = context, + dbName = null, + indexStrategy = indexStrategy, + ) + } + } + } + } } @After diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt index 9d1ffc3c3..0a0d9b90a 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt @@ -208,26 +208,32 @@ class DeletionTest : BaseDBTest() { @Test fun testTriggersIndexUsage() = forEachDB { db -> - val sql = - """ - SELECT 1 FROM event_tags - WHERE - event_tags.tag_hash IN (3221122, 223322) AND - event_tags.kind = 5 AND - event_tags.pubkey_hash = 22332323 AND - event_tags.created_at >= 1766686500 - """.trimIndent() + var sql = db.store.deletionModule.rejectDeletedEventsSQLTemplate() - val explainer = - db.store.explainQuery(sql) + sql = sql.replace("NEW.etag_hash", "3221122") + sql = sql.replace("NEW.atag_hash", "223322") + sql = sql.replace("NEW.pubkey_owner_hash", "22332323") + sql = sql.replace("NEW.created_at", "1766686500") - TestCase.assertEquals( - """ - ${sql.replace("\n","\n ")} - └── SEARCH event_tags USING COVERING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?) - """.trimIndent(), - explainer, - ) + val explainer = db.store.explainQuery(sql) + + if (db.indexStrategy.indexTagsWithKindAndPubkey) { + TestCase.assertEquals( + """ + |$sql + |└── SEARCH event_tags USING COVERING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?) + """.trimMargin(), + explainer, + ) + } else { + TestCase.assertEquals( + """ + |$sql + |└── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=? AND created_at>?) + """.trimMargin(), + explainer, + ) + } } @Test diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt index 52d4c5423..3dd656c27 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -20,8 +20,6 @@ */ package com.vitorpamplona.quartz.nip01Core.store.sqlite -import android.content.Context -import androidx.test.core.app.ApplicationProvider import com.vitorpamplona.quartz.experimental.relationshipStatus.ContactCardEvent import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter @@ -32,665 +30,991 @@ import com.vitorpamplona.quartz.nip50Search.SearchRelayListEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent import junit.framework.TestCase import junit.framework.TestCase.assertEquals -import org.junit.After import org.junit.Assert -import org.junit.Before import org.junit.Test -class QueryAssemblerTest { +class QueryAssemblerTest : BaseDBTest() { val hasher = TagNameValueHasher(0) val key1 = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" val key2 = "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14" val key3 = "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9" - private lateinit var dbAllIndexes: EventStore - private lateinit var dbNoKindIndexes: EventStore - - @Before - fun setup() { - val context = ApplicationProvider.getApplicationContext() - dbAllIndexes = EventStore(context, null, indexStrategy = DefaultIndexingStrategy(true)) - dbNoKindIndexes = EventStore(context, null, indexStrategy = DefaultIndexingStrategy(false)) - } - - @After - fun tearDown() { - dbAllIndexes.close() - dbNoKindIndexes.close() - } - fun EventStore.explain(f: Filter) = store.queryBuilder.planQuery(f, hasher, store.readableDatabase) fun EventStore.explain(f: List) = store.queryBuilder.planQuery(f, hasher, store.readableDatabase) @Test - fun testEmpty() { - Assert.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - ORDER BY created_at DESC - └── SCAN event_headers USING INDEX query_by_created_at_id - """.trimIndent(), - dbAllIndexes.explain(Filter()), - ) - - Assert.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - ORDER BY created_at DESC - └── SCAN event_headers USING INDEX query_by_created_at_id - """.trimIndent(), - dbNoKindIndexes.explain(Filter()), - ) - } + fun testEmpty() = + forEachDB { db -> + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + if (db.indexStrategy.indexEventsByCreatedAtAlone) { + Assert.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + ORDER BY $orderBy + └── SCAN event_headers USING INDEX query_by_created_at_id + """.trimIndent(), + db.explain(Filter()), + ) + } else { + Assert.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + ORDER BY $orderBy + ├── SCAN event_headers + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(Filter()), + ) + } + } @Test - fun testCheckDeletionEventExists() { - val filter = - Filter( - kinds = listOf(5), - authors = listOf(key1), - tags = mapOf("e" to listOf(key2)), - since = 1750889190, - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "2657743813502222172") AND (event_tags.kind = "5") AND (event_tags.pubkey_hash = "1730514094536529999") AND (event_tags.created_at >= "1750889190") - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "2657743813502222172") AND (event_tags.kind = "5") AND (event_tags.pubkey_hash = "1730514094536529999") AND (event_tags.created_at >= "1750889190") - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbNoKindIndexes.explain(filter), - ) - } - - @Test - fun testLimit() { - val filter = Filter(limit = 10) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - ORDER BY created_at DESC - LIMIT 10 - └── SCAN event_headers USING INDEX query_by_created_at_id - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testLimits() { - val filter = listOf(Filter(limit = 10), Filter(limit = 30)) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) - UNION - SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 30) - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ └── COMPOUND QUERY - │ ├── LEFT-MOST SUBQUERY - │ │ ├── CO-ROUTINE (subquery-1) - │ │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id - │ │ └── SCAN (subquery-1) - │ └── UNION USING TEMP B-TREE - │ ├── CO-ROUTINE (subquery-3) - │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id - │ └── SCAN (subquery-3) - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testAllFeatures() { - val filter = - listOf( - Filter(limit = 10), + fun testCheckDeletionEventExists() = + forEachDB { db -> + val filter = Filter( + kinds = listOf(5), authors = listOf(key1), - kinds = listOf(1, 1111), - search = "keywords", - limit = 100, - ), - Filter(kinds = listOf(20), search = "cats", limit = 30), - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) - UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) - UNION - SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ └── COMPOUND QUERY - │ ├── LEFT-MOST SUBQUERY - │ │ ├── CO-ROUTINE (subquery-1) - │ │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id - │ │ └── SCAN (subquery-1) - │ ├── UNION USING TEMP B-TREE - │ │ ├── CO-ROUTINE (subquery-3) - │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 4: - │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - │ │ │ └── USE TEMP B-TREE FOR ORDER BY - │ │ └── SCAN (subquery-3) - │ └── UNION USING TEMP B-TREE - │ ├── CO-ROUTINE (subquery-5) - │ │ ├── SEARCH event_headers USING COVERING INDEX query_by_kind_created (kind=?) - │ │ └── SCAN event_fts VIRTUAL TABLE INDEX 4: - │ └── SCAN (subquery-5) - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } + tags = mapOf("e" to listOf(key2)), + since = 1750889190, + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + if (db.indexStrategy.indexTagsWithKindAndPubkey) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "2657743813502222172") AND (event_tags.kind = "5") AND (event_tags.pubkey_hash = "1730514094536529999") AND (event_tags.created_at >= "1750889190") + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "2657743813502222172") AND (event_tags.kind = "5") AND (event_tags.pubkey_hash = "1730514094536529999") AND (event_tags.created_at >= "1750889190") + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=? AND created_at>?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } @Test - fun testSingleFilterConversionToSimpleQuery() { - val filter = - listOf( + fun testLimit() = + forEachDB { db -> + val filter = Filter(limit = 10) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + + if (db.indexStrategy.indexEventsByCreatedAtAlone) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + ORDER BY $orderBy + LIMIT 10 + └── SCAN event_headers USING INDEX query_by_created_at_id + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + ORDER BY $orderBy + LIMIT 10 + ├── SCAN event_headers + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testLimits() = + forEachDB { db -> + val filter = listOf(Filter(limit = 10), Filter(limit = 30)) + val orderBy = + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + "created_at DESC, id ASC" + } else { + "created_at DESC" + } + if (db.indexStrategy.indexEventsByCreatedAtAlone) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) + UNION + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 30) + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ └── COMPOUND QUERY + │ ├── LEFT-MOST SUBQUERY + │ │ ├── CO-ROUTINE (subquery-1) + │ │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id + │ │ └── SCAN (subquery-1) + │ └── UNION USING TEMP B-TREE + │ ├── CO-ROUTINE (subquery-3) + │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id + │ └── SCAN (subquery-3) + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) + UNION + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 30) + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ └── COMPOUND QUERY + │ ├── LEFT-MOST SUBQUERY + │ │ ├── CO-ROUTINE (subquery-1) + │ │ │ ├── SCAN event_headers USING COVERING INDEX query_by_kind_created + │ │ │ └── USE TEMP B-TREE FOR ORDER BY + │ │ └── SCAN (subquery-1) + │ └── UNION USING TEMP B-TREE + │ ├── CO-ROUTINE (subquery-3) + │ │ ├── SCAN event_headers USING COVERING INDEX query_by_kind_created + │ │ └── USE TEMP B-TREE FOR ORDER BY + │ └── SCAN (subquery-3) + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testAllFeatures() = + forEachDB { db -> + val filter = + listOf( + Filter(limit = 10), + Filter( + authors = listOf(key1), + kinds = listOf(1, 1111), + search = "keywords", + limit = 100, + ), + Filter(kinds = listOf(20), search = "cats", limit = 30), + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + if (db.indexStrategy.indexEventsByCreatedAtAlone) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) + UNION + SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) + UNION + SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ └── COMPOUND QUERY + │ ├── LEFT-MOST SUBQUERY + │ │ ├── CO-ROUTINE (subquery-1) + │ │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id + │ │ └── SCAN (subquery-1) + │ ├── UNION USING TEMP B-TREE + │ │ ├── CO-ROUTINE (subquery-3) + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ │ │ └── USE TEMP B-TREE FOR ORDER BY + │ │ └── SCAN (subquery-3) + │ └── UNION USING TEMP B-TREE + │ ├── CO-ROUTINE (subquery-5) + │ │ ├── SEARCH event_headers USING COVERING INDEX query_by_kind_created (kind=?) + │ │ └── SCAN event_fts VIRTUAL TABLE INDEX 4: + │ └── SCAN (subquery-5) + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers ORDER BY event_headers.created_at DESC LIMIT 10) + UNION + SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC LIMIT 100) + UNION + SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC LIMIT 30) + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ └── COMPOUND QUERY + │ ├── LEFT-MOST SUBQUERY + │ │ ├── CO-ROUTINE (subquery-1) + │ │ │ ├── SCAN event_headers USING COVERING INDEX query_by_kind_created + │ │ │ └── USE TEMP B-TREE FOR ORDER BY + │ │ └── SCAN (subquery-1) + │ ├── UNION USING TEMP B-TREE + │ │ ├── CO-ROUTINE (subquery-3) + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ │ │ └── USE TEMP B-TREE FOR ORDER BY + │ │ └── SCAN (subquery-3) + │ └── UNION USING TEMP B-TREE + │ ├── CO-ROUTINE (subquery-5) + │ │ ├── SEARCH event_headers USING COVERING INDEX query_by_kind_created (kind=?) + │ │ └── SCAN event_fts VIRTUAL TABLE INDEX 4: + │ └── SCAN (subquery-5) + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testSingleFilterConversionToSimpleQuery() = + forEachDB { db -> + val filter = + listOf( + Filter( + kinds = listOf(ContactListEvent.KIND), + limit = 30, + ), + ) + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE kind = "3" + ORDER BY created_at DESC, id ASC + LIMIT 30 + └── SEARCH event_headers USING INDEX query_by_kind_created (kind=?) + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE kind = "3" + ORDER BY created_at DESC + LIMIT 30 + └── SEARCH event_headers USING INDEX query_by_kind_created (kind=?) + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testKinds() = + forEachDB { db -> + val filter = + listOf( + Filter( + kinds = listOf(ContactListEvent.KIND), + limit = 30, + ), + Filter( + kinds = listOf(TextNoteEvent.KIND), + limit = 30, + ), + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers WHERE event_headers.kind = "3" ORDER BY event_headers.created_at DESC LIMIT 30) + UNION + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers WHERE event_headers.kind = "1" ORDER BY event_headers.created_at DESC LIMIT 30) + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ └── COMPOUND QUERY + │ ├── LEFT-MOST SUBQUERY + │ │ ├── CO-ROUTINE (subquery-1) + │ │ │ └── SEARCH event_headers USING COVERING INDEX query_by_kind_created (kind=?) + │ │ └── SCAN (subquery-1) + │ └── UNION USING TEMP B-TREE + │ ├── CO-ROUTINE (subquery-3) + │ │ └── SEARCH event_headers USING COVERING INDEX query_by_kind_created (kind=?) + │ └── SCAN (subquery-3) + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + + @Test + fun testKindAndDTag() = + forEachDB { db -> + val filter = + listOf( + Filter( + kinds = listOf(ContactListEvent.KIND), + tags = mapOf("d" to listOf("")), + limit = 30, + ), + ) + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind = "3") AND (d_tag = "") + ORDER BY created_at DESC, id ASC + LIMIT 30 + └── SEARCH event_headers USING INDEX query_by_kind_created (kind=?) + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind = "3") AND (d_tag = "") + ORDER BY created_at DESC + LIMIT 30 + └── SEARCH event_headers USING INDEX query_by_kind_created (kind=?) + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testFollowersOf() = + forEachDB { db -> + val filter = + listOf( + Filter( + kinds = listOf(ContactListEvent.KIND), + tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), + limit = 30, + ), + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") ORDER BY event_tags.created_at DESC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + + @Test + fun testNotificationsOf() = + forEachDB { db -> + val filter = + listOf( + Filter( + tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), + limit = 30, + ), + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + if (db.indexStrategy.indexTagsByCreatedAtAlone) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE event_tags.tag_hash = "-4551135004136952885" ORDER BY event_tags.created_at DESC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE event_tags.tag_hash = "-4551135004136952885" ORDER BY event_tags.created_at DESC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=?) + │ ├── USE TEMP B-TREE FOR DISTINCT + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testTagsAndKinds() = + forEachDB { db -> + val filter = + listOf( + Filter( + kinds = listOf(ContactListEvent.KIND), + tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), + limit = 30, + ), + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") ORDER BY event_tags.created_at DESC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + + @Test + fun testTagsAndAuthors() = + forEachDB { db -> + val filter = + listOf( + Filter( + authors = listOf(key1), + tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), + limit = 30, + ), + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + if (db.indexStrategy.indexTagsByCreatedAtAlone) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.pubkey_hash = "1730514094536529999") ORDER BY event_tags.created_at DESC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.pubkey_hash = "1730514094536529999") ORDER BY event_tags.created_at DESC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=?) + │ ├── USE TEMP B-TREE FOR DISTINCT + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testTwoTags() = + forEachDB { db -> + val filter = + listOf( + Filter( + kinds = listOf(1), + tags = + mapOf( + "p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), + "t" to listOf("hashtag"), + ), + limit = 30, + ), + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + if (db.indexStrategy.indexTagsByCreatedAtAlone) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsIn1 ON event_tagsIn1.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn1.created_at = event_tags.created_at WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tagsIn1.tag_hash = "-6379614208644810021") AND (event_tags.kind = "1") ORDER BY event_tags.created_at DESC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=?) + │ ├── SEARCH event_tagsIn1 USING INDEX query_by_tags_hash (tag_hash=? AND created_at=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsIn1 ON event_tagsIn1.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn1.created_at = event_tags.created_at WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tagsIn1.tag_hash = "-6379614208644810021") AND (event_tags.kind = "1") ORDER BY event_tags.created_at DESC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=?) + │ ├── SEARCH event_tagsIn1 USING INDEX fk_event_tags_header_id (event_header_row_id=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testIdQuery() = + forEachDB { db -> + val filter = Filter(ids = listOf(key1)) + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE id = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" + ORDER BY created_at DESC, id ASC + └── SEARCH event_headers USING INDEX event_headers_id (id=?) + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE id = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" + ORDER BY created_at DESC + └── SEARCH event_headers USING INDEX event_headers_id (id=?) + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testAuthors() = + forEachDB { db -> + val filter = Filter(authors = listOf(key1, key2), kinds = listOf(1, 30023), limit = 300) + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind IN ("1", "30023")) AND (pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14")) + ORDER BY created_at DESC, id ASC + LIMIT 300 + ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind IN ("1", "30023")) AND (pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14")) + ORDER BY created_at DESC + LIMIT 300 + ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testAuthorsAndSearch() = + forEachDB { db -> + val filter = Filter(authors = listOf(key1, key2, key3), search = "keywords") + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + TestCase.assertEquals( + """ + SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers + INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) + ORDER BY event_headers.created_at DESC, event_headers.id ASC + ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers + INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) + ORDER BY event_headers.created_at DESC + ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testKindAndSearch() = + forEachDB { db -> + val filter = Filter(kinds = listOf(1, 1111, 10000), search = "keywords") + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "event_headers.created_at DESC, event_headers.id ASC" else "event_headers.created_at DESC" + TestCase.assertEquals( + """ + SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers + INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id + WHERE (event_fts MATCH "keywords") AND (event_headers.kind IN ("1", "1111", "10000")) + ORDER BY $orderBy + ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + + @Test + fun testAllTag() = + forEachDB { db -> + val filter = Filter(tagsAll = mapOf("p" to listOf(key1, key2))) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + if (db.indexStrategy.indexTagsByCreatedAtAlone) { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsAll0_1 ON event_tagsAll0_1.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll0_1.created_at = event_tags.created_at WHERE (event_tags.tag_hash = "884286737453847614") AND (event_tagsAll0_1.tag_hash = "-4988851810256311323") + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=?) + │ ├── SEARCH event_tagsAll0_1 USING INDEX query_by_tags_hash (tag_hash=? AND created_at=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsAll0_1 ON event_tagsAll0_1.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll0_1.created_at = event_tags.created_at WHERE (event_tags.tag_hash = "884286737453847614") AND (event_tagsAll0_1.tag_hash = "-4988851810256311323") + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=?) + │ ├── SEARCH event_tagsAll0_1 USING INDEX fk_event_tags_header_id (event_header_row_id=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testReportLikeFilter() = + forEachDB { db -> + val filter = Filter( kinds = listOf(ContactListEvent.KIND), - limit = 30, - ), - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - WHERE kind = "3" - ORDER BY created_at DESC - LIMIT 30 - └── SEARCH event_headers USING INDEX query_by_kind_created (kind=?) - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testKinds() { - val filter = - listOf( - Filter( - kinds = listOf(ContactListEvent.KIND), - limit = 30, - ), - Filter( - kinds = listOf(TextNoteEvent.KIND), - limit = 30, - ), - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers WHERE event_headers.kind = "3" ORDER BY event_headers.created_at DESC LIMIT 30) - UNION - SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers WHERE event_headers.kind = "1" ORDER BY event_headers.created_at DESC LIMIT 30) - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ └── COMPOUND QUERY - │ ├── LEFT-MOST SUBQUERY - │ │ ├── CO-ROUTINE (subquery-1) - │ │ │ └── SEARCH event_headers USING COVERING INDEX query_by_kind_created (kind=?) - │ │ └── SCAN (subquery-1) - │ └── UNION USING TEMP B-TREE - │ ├── CO-ROUTINE (subquery-3) - │ │ └── SEARCH event_headers USING COVERING INDEX query_by_kind_created (kind=?) - │ └── SCAN (subquery-3) - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testKindAndDTag() { - val filter = - listOf( - Filter( - kinds = listOf(ContactListEvent.KIND), - tags = mapOf("d" to listOf("")), - limit = 30, - ), - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - WHERE (kind = "3") AND (d_tag = "") - ORDER BY created_at DESC - LIMIT 30 - └── SEARCH event_headers USING INDEX query_by_kind_created (kind=?) - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testFollowersOf() { - val filter = - listOf( - Filter( - kinds = listOf(ContactListEvent.KIND), - tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), - limit = 30, - ), - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") ORDER BY event_tags.created_at DESC LIMIT 30 - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testNotificationsOf() { - val filter = - listOf( - Filter( - tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), - limit = 30, - ), - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE event_tags.tag_hash = "-4551135004136952885" ORDER BY event_tags.created_at DESC LIMIT 30 - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testTagsAndKinds() { - val filter = - listOf( - Filter( - kinds = listOf(ContactListEvent.KIND), - tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), - limit = 30, - ), - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") ORDER BY event_tags.created_at DESC LIMIT 30 - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testTagsAndAuthors() { - val filter = - listOf( - Filter( - authors = listOf(key1), - tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), - limit = 30, - ), - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.pubkey_hash = "1730514094536529999") ORDER BY event_tags.created_at DESC LIMIT 30 - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testTwoTags() { - val filter = - listOf( - Filter( - kinds = listOf(1), + authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), tags = mapOf( "p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - "t" to listOf("hashtag"), ), - limit = 30, - ), + limit = 500, + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + if (db.indexStrategy.indexTagsWithKindAndPubkey) { + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") AND (event_tags.pubkey_hash = "5446767199141196776") ORDER BY event_tags.created_at DESC LIMIT 500 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") AND (event_tags.pubkey_hash = "5446767199141196776") ORDER BY event_tags.created_at DESC LIMIT 500 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } + + @Test + fun testFollowersSinceNov2025() = + forEachDB { db -> + val filter = + Filter( + kinds = listOf(ContactListEvent.KIND), + tags = + mapOf( + "p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), + ), + since = 1764553447, // Nov 2025 + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") AND (event_tags.created_at >= "1764553447") + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY $orderBy + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=? AND created_at>?) + │ └── USE TEMP B-TREE FOR DISTINCT + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsIn1 ON event_tagsIn1.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn1.created_at = event_tags.created_at WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tagsIn1.tag_hash = "-6379614208644810021") AND (event_tags.kind = "1") ORDER BY event_tags.created_at DESC LIMIT 30 - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=?) - │ ├── SEARCH event_tagsIn1 USING INDEX query_by_tags_hash (tag_hash=? AND created_at=?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } + } @Test - fun testIdQuery() { - val filter = Filter(ids = listOf(key1)) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - WHERE id = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" - ORDER BY created_at DESC - └── SEARCH event_headers USING INDEX event_headers_id (id=?) - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testAuthors() { - val filter = Filter(authors = listOf(key1, key2), kinds = listOf(1, 30023), limit = 300) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - WHERE (kind IN ("1", "30023")) AND (pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14")) - ORDER BY created_at DESC - LIMIT 300 - ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testAuthorsAndSearch() { - val filter = Filter(authors = listOf(key1, key2, key3), search = "keywords") - println(dbAllIndexes.explain(filter)) - TestCase.assertEquals( - """ - SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id - WHERE (event_fts MATCH "keywords") AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) - ORDER BY event_headers.created_at DESC - ├── SCAN event_fts VIRTUAL TABLE INDEX 4: - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testKindAndSearch() { - val filter = Filter(kinds = listOf(1, 1111, 10000), search = "keywords") - println(dbAllIndexes.explain(filter)) - TestCase.assertEquals( - """ - SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers - INNER JOIN event_fts ON event_headers.row_id = event_fts.event_header_row_id - WHERE (event_fts MATCH "keywords") AND (event_headers.kind IN ("1", "1111", "10000")) - ORDER BY event_headers.created_at DESC - ├── SCAN event_fts VIRTUAL TABLE INDEX 4: - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testAllTag() { - val filter = Filter(tagsAll = mapOf("p" to listOf(key1, key2))) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsAll0_1 ON event_tagsAll0_1.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll0_1.created_at = event_tags.created_at WHERE (event_tags.tag_hash = "884286737453847614") AND (event_tagsAll0_1.tag_hash = "-4988851810256311323") - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=?) - │ ├── SEARCH event_tagsAll0_1 USING INDEX query_by_tags_hash (tag_hash=? AND created_at=?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testReportLikeFilter() { - val filter = - Filter( - kinds = listOf(ContactListEvent.KIND), - authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - tags = - mapOf( - "p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - ), - limit = 500, + fun testAllAddressablesOfAKindDownload() = + forEachDB { db -> + val filter = + Filter( + kinds = listOf(DraftWrapEvent.KIND), + authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), + since = 1764553447, // Nov 2025 + ) + val orderBy = if (db.indexStrategy.useAndIndexIdOnOrderBy) "created_at DESC, id ASC" else "created_at DESC" + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind = "31234") AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (created_at >= "1764553447") + ORDER BY $orderBy + └── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=? AND created_at>?) + """.trimIndent(), + db.explain(filter), ) - assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") AND (event_tags.pubkey_hash = "5446767199141196776") ORDER BY event_tags.created_at DESC LIMIT 500 - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } + } @Test - fun testFollowersSinceNov2025() { - val filter = - Filter( - kinds = listOf(ContactListEvent.KIND), - tags = - mapOf( - "p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - ), - since = 1764553447, // Nov 2025 - ) - assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tags.kind = "3") AND (event_tags.created_at >= "1764553447") - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC - ├── CO-ROUTINE filtered - │ ├── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=? AND created_at>?) - │ └── USE TEMP B-TREE FOR DISTINCT - ├── SCAN filtered - ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } + fun testReplaceablesOfMultipleKindsDownloadByDateLimit() = + forEachDB { db -> + val filter = + Filter( + kinds = listOf(MetadataEvent.KIND, SearchRelayListEvent.KIND, AdvertisedRelayListEvent.KIND), + authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), + limit = 20, + since = 1764553447, // Nov 2025 + ) + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind IN ("0", "10007", "10002")) AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (created_at >= "1764553447") + ORDER BY created_at DESC, id ASC + LIMIT 20 + ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=? AND created_at>?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind IN ("0", "10007", "10002")) AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (created_at >= "1764553447") + ORDER BY created_at DESC + LIMIT 20 + ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=? AND created_at>?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } @Test - fun testAllAddressablesOfAKindDownload() { - val filter = - Filter( - kinds = listOf(DraftWrapEvent.KIND), - authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - since = 1764553447, // Nov 2025 - ) - assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - WHERE (kind = "31234") AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (created_at >= "1764553447") - ORDER BY created_at DESC - └── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=? AND created_at>?) - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } + fun testReplaceablesOfMultipleKindsDownload() = + forEachDB { db -> + val filter = + Filter( + kinds = listOf(MetadataEvent.KIND, SearchRelayListEvent.KIND, AdvertisedRelayListEvent.KIND), + authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), + ) + + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind IN ("0", "10007", "10002")) AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") + ORDER BY created_at DESC, id ASC + ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind IN ("0", "10007", "10002")) AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") + ORDER BY created_at DESC + ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } @Test - fun testReplaceablesOfMultipleKindsDownloadByDateLimit() { - val filter = - Filter( - kinds = listOf(MetadataEvent.KIND, SearchRelayListEvent.KIND, AdvertisedRelayListEvent.KIND), - authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - limit = 20, - since = 1764553447, // Nov 2025 - ) - assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - WHERE (kind IN ("0", "10007", "10002")) AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (created_at >= "1764553447") - ORDER BY created_at DESC - LIMIT 20 - ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=? AND created_at>?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } + fun testContactCardDownloadFromTrustedKeys() = + forEachDB { db -> + val filter = + Filter( + kinds = listOf(ContactCardEvent.KIND), + authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), + tags = + mapOf( + "d" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), + ), + since = 1764553447, // Nov 2025 + ) - @Test - fun testReplaceablesOfMultipleKindsDownload() { - val filter = - Filter( - kinds = listOf(MetadataEvent.KIND, SearchRelayListEvent.KIND, AdvertisedRelayListEvent.KIND), - authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - ) - - assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - WHERE (kind IN ("0", "10007", "10002")) AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") - ORDER BY created_at DESC - ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_created (kind=? AND pubkey=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } - - @Test - fun testContactCardDownloadFromTrustedKeys() { - val filter = - Filter( - kinds = listOf(ContactCardEvent.KIND), - authors = listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - tags = - mapOf( - "d" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), - ), - since = 1764553447, // Nov 2025 - ) - - assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - WHERE (kind = "30382") AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (d_tag = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (created_at >= "1764553447") AND ((kind >= 30000 AND kind < 40000)) - ORDER BY created_at DESC - ├── SEARCH event_headers USING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?) - └── USE TEMP B-TREE FOR ORDER BY - """.trimIndent(), - dbAllIndexes.explain(filter), - ) - } + if (db.indexStrategy.useAndIndexIdOnOrderBy) { + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind = "30382") AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (d_tag = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (created_at >= "1764553447") AND ((kind >= 30000 AND kind < 40000)) + ORDER BY created_at DESC, id ASC + ├── SEARCH event_headers USING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } else { + assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + WHERE (kind = "30382") AND (pubkey = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (d_tag = "460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c") AND (created_at >= "1764553447") AND ((kind >= 30000 AND kind < 40000)) + ORDER BY created_at DESC + ├── SEARCH event_headers USING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + db.explain(filter), + ) + } + } } diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt index deb4a061d..6905889e6 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt @@ -30,32 +30,45 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent class DeletionRequestModule( val hasher: (db: SQLiteDatabase) -> TagNameValueHasher, + val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(), ) : IModule { + fun rejectDeletedEventsSQLTemplate(): String = + if (indexStrategy.indexTagsWithKindAndPubkey) { + """ + |SELECT 1 FROM event_tags + |WHERE + | event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND + | event_tags.kind = 5 AND + | event_tags.pubkey_hash = NEW.pubkey_owner_hash AND + | event_tags.created_at >= NEW.created_at + """.trimMargin() + } else { + """ + |SELECT 1 FROM event_tags + |WHERE + | event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND + | event_tags.kind = 5 AND + | event_tags.created_at >= NEW.created_at AND + | event_tags.pubkey_hash = NEW.pubkey_owner_hash + """.trimMargin() + } + /** * Creates a trigger to reject events that have been * deleted by ID or ATag including GiftWraps that * must be checked against the p-tag (pubkey_owner_hash) */ override fun create(db: SQLiteDatabase) { + val sql = rejectDeletedEventsSQLTemplate().replace("\n", "\n ") db.execSQL( """ CREATE TRIGGER reject_deleted_events BEFORE INSERT ON event_headers FOR EACH ROW BEGIN - -- Checks if this event hasn't already been deleted. If so, reject it. - - -- Highly optimized using hash + created_at (duplicated in tags) index: - ---- Expects tag_hash to be non-existant for new events (quick exit) - ---- Expects created-at >= event's to not exist (quick exit) SELECT RAISE(ABORT, 'blocked: a deletion event exists') WHERE EXISTS ( - SELECT 1 FROM event_tags - WHERE - event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND - event_tags.kind = 5 AND - event_tags.pubkey_hash = NEW.pubkey_owner_hash AND - event_tags.created_at >= NEW.created_at + $sql ); END; """.trimIndent(), diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt index 4dbe73aa4..8aefa83e4 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt @@ -75,8 +75,9 @@ class EventIndexesModule( } // queries by limit (latest records), since, until (sync all) alone without any filter by kind.. rare - // serves as fall back for the lack of query_by_tags_hash index by default - db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers ($orderBy)") + if (indexStrategy.indexEventsByCreatedAtAlone) { + db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers ($orderBy)") + } // queries by kind only, mostly used in Global Feeds when author is not important. db.execSQL("CREATE INDEX query_by_kind_created ON event_headers (kind, $orderBy)") @@ -87,10 +88,10 @@ class EventIndexesModule( // makes deletions on the event_header fast db.execSQL("CREATE INDEX fk_event_tags_header_id ON event_tags (event_header_row_id)") - // --------------------------------------------------------------- - // This are a very slow indexes (80% of the insert time goes here) - // --------------------------------------------------------------- - if (indexStrategy.indexTagQueriesWithoutKinds) { + // --------------------------------------------------------------------------- + // These next 3 are a very slow indexes (80% of the insert time goes here) + // --------------------------------------------------------------------------- + if (indexStrategy.indexTagsByCreatedAtAlone) { // First one is only needed if the user is searching by tags without a kind. db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, created_at DESC)") } @@ -98,8 +99,10 @@ class EventIndexesModule( // This is the default index for most clients: tags by specific kinds that are supported by the client. db.execSQL("CREATE INDEX query_by_tags_hash_kind ON event_tags (tag_hash, kind, created_at DESC)") - // this one is to allow search of tags by kind and author at the same time: DMs, reports, - db.execSQL("CREATE INDEX query_by_tags_hash_kind_pubkey ON event_tags (tag_hash, kind, pubkey_hash, created_at DESC)") + // this one is to allow search of tags by kind and author at the same time: NIP-04 DMs, reports, + if (indexStrategy.indexTagsWithKindAndPubkey) { + db.execSQL("CREATE INDEX query_by_tags_hash_kind_pubkey ON event_tags (tag_hash, kind, pubkey_hash, created_at DESC)") + } // Prevent updates to maintain immutability db.execSQL( diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt index 1f5cf7745..a0a6ea1d4 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt @@ -23,7 +23,62 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import com.vitorpamplona.quartz.nip01Core.core.Tag interface IndexingStrategy { - val indexTagQueriesWithoutKinds: Boolean + /** + * Activate this if you see too many Filters with just LIMIT, SINCE and + * UNTIL filled up. + * + * Clients never support all kinds, so this is usually + * only done with syncing services that must download ALL kinds from + * ALL authors. + * + * The index will make these queries significantly faster, but maybe speed + * is not a requirement on Sync services. The size of this index is + * considerable. + * + * Keep in mind that activating too many indexes increases the size of the + * DB so much that the indexes themselves won't fit in memory, requiring + * frequent reloadings of the index itself from disk. + */ + val indexEventsByCreatedAtAlone: Boolean + + /** + * Activate this if you see too many Tag-centric Filters without + * kind, pubkey or id. + * + * Clients never support all kinds, so this is usually + * only done in rare usecases where the client supports all + * kinds. + * + * The index will make these queries significantly faster, but maybe speed + * is not a requirement on such services. Because this is an index in + * event tags, it becomes QUITE BIG. + * + * Keep in mind that activating too many indexes increases the size of the + * DB so much that the indexes themselves won't fit in memory, requiring + * frequent reloadings of the index itself from disk. + */ + val indexTagsByCreatedAtAlone: Boolean + + /** + * Activate this if you see too many Tag-centric Filters without + * kind AND pubkey at the same time. + * + * This is a rarely used index (reports by your follows or + * NIP-04 DMs for instance) that becomes quite large without + * major gains. + * + * Keep in mind that activating too many indexes increases the size of the + * DB so much that the indexes themselves won't fit in memory, requiring + * frequent reloadings of the index itself from disk. + */ + val indexTagsWithKindAndPubkey: Boolean + + /** + * Activate this to make sure queries are always in order when + * the same created_at exists. This will impact performance and + * the size of indexes, but it provides results that are compliant + * with the Nostr Spec + */ val useAndIndexIdOnOrderBy: Boolean fun shouldIndex( @@ -36,7 +91,9 @@ interface IndexingStrategy { * By default, we index all tags that have a single letter name and some value */ class DefaultIndexingStrategy( - override val indexTagQueriesWithoutKinds: Boolean = false, + override val indexEventsByCreatedAtAlone: Boolean = false, + override val indexTagsByCreatedAtAlone: Boolean = false, + override val indexTagsWithKindAndPubkey: Boolean = false, override val useAndIndexIdOnOrderBy: Boolean = false, ) : IndexingStrategy { override fun shouldIndex( diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt index ef4467b84..45e797cb4 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -655,7 +655,7 @@ class QueryBuilder( if (project) { append("\nORDER BY created_at DESC") if (indexStrategy.useAndIndexIdOnOrderBy) { - append(", event_headers.id ASC") + append(", id ASC") } } if (limit != null) {