Creating additional configuration options and amplifying tests to a combination of all those options

quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt

@@ -26,19 +26,46 @@ import org.junit.After
 import org.junit.Before
 
 open class BaseDBTest {
-    private lateinit var dbs: Map<String, EventStore>
+    private lateinit var dbs: MutableMap<String, EventStore>
+
+    fun DefaultIndexingStrategy.name(): String =
+        """
+        indexEventsByCreatedAtAlone=$indexEventsByCreatedAtAlone
+        indexTagsByCreatedAtAlone=$indexTagsByCreatedAtAlone
+        indexTagsWithKindAndPubkey=$indexTagsWithKindAndPubkey
+        useAndIndexIdOnOrderBy=$useAndIndexIdOnOrderBy
+        """.trimIndent()
 
     @Before
     fun setup() {
         val context = ApplicationProvider.getApplicationContext<Context>()
 
-        dbs =
+        val booleans = listOf(true, false)
-            mapOf(
+
-                "Default" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(false, false)),
+        dbs = mutableMapOf<String, EventStore>()
-                "IndexAll" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(true, false)),
+
-                "Order by ID" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(false, true)),
+        // tests all possible DBs
-                "IndexAll, Order by ID" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(true, true)),
+        for (indexEventsByCreatedAtAlone in booleans) {
-            )
+            for (indexTagsByCreatedAtAlone in booleans) {
+                for (indexTagsWithKindAndPubkey in booleans) {
+                    for (useAndIndexIdOnOrderBy in booleans) {
+                        val indexStrategy =
+                            DefaultIndexingStrategy(
+                                indexEventsByCreatedAtAlone,
+                                indexTagsByCreatedAtAlone,
+                                indexTagsWithKindAndPubkey,
+                                useAndIndexIdOnOrderBy,
+                            )
+                        dbs[indexStrategy.name()] =
+                            EventStore(
+                                context = context,
+                                dbName = null,
+                                indexStrategy = indexStrategy,
+                            )
+                    }
+                }
+            }
+        }
     }
 
     @After

quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt

@@ -208,26 +208,32 @@ class DeletionTest : BaseDBTest() {
     @Test
     fun testTriggersIndexUsage() =
         forEachDB { db ->
-            val sql =
+            var sql = db.store.deletionModule.rejectDeletedEventsSQLTemplate()
-                """
-                SELECT 1 FROM event_tags
-                WHERE
-                    event_tags.tag_hash IN (3221122, 223322) AND
-                    event_tags.kind = 5 AND
-                    event_tags.pubkey_hash = 22332323 AND
-                    event_tags.created_at >= 1766686500
-                """.trimIndent()
 
-            val explainer =
+            sql = sql.replace("NEW.etag_hash", "3221122")
-                db.store.explainQuery(sql)
+            sql = sql.replace("NEW.atag_hash", "223322")
+            sql = sql.replace("NEW.pubkey_owner_hash", "22332323")
+            sql = sql.replace("NEW.created_at", "1766686500")
 
-            TestCase.assertEquals(
+            val explainer = db.store.explainQuery(sql)
-                """
+
-                ${sql.replace("\n","\n                ")}
+            if (db.indexStrategy.indexTagsWithKindAndPubkey) {
-                └── SEARCH event_tags USING COVERING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?)
+                TestCase.assertEquals(
-                """.trimIndent(),
+                    """
-                explainer,
+                    |$sql
-            )
+                    |└── SEARCH event_tags USING COVERING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?)
+                    """.trimMargin(),
+                    explainer,
+                )
+            } else {
+                TestCase.assertEquals(
+                    """
+                    |$sql
+                    |└── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=? AND created_at>?)
+                    """.trimMargin(),
+                    explainer,
+                )
+            }
         }
 
     @Test

quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt

@@ -30,32 +30,45 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
 
 class DeletionRequestModule(
     val hasher: (db: SQLiteDatabase) -> TagNameValueHasher,
+    val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
 ) : IModule {
+    fun rejectDeletedEventsSQLTemplate(): String =
+        if (indexStrategy.indexTagsWithKindAndPubkey) {
+            """
+            |SELECT 1 FROM event_tags
+            |WHERE
+            |    event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND
+            |    event_tags.kind = 5 AND
+            |    event_tags.pubkey_hash = NEW.pubkey_owner_hash AND
+            |    event_tags.created_at >= NEW.created_at
+            """.trimMargin()
+        } else {
+            """
+            |SELECT 1 FROM event_tags
+            |WHERE
+            |    event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND
+            |    event_tags.kind = 5 AND
+            |    event_tags.created_at >= NEW.created_at AND
+            |    event_tags.pubkey_hash = NEW.pubkey_owner_hash
+            """.trimMargin()
+        }
+
     /**
      * Creates a trigger to reject events that have been
      * deleted by ID or ATag including GiftWraps that
      * must be checked against the p-tag (pubkey_owner_hash)
      */
     override fun create(db: SQLiteDatabase) {
+        val sql = rejectDeletedEventsSQLTemplate().replace("\n", "\n                    ")
         db.execSQL(
             """
             CREATE TRIGGER reject_deleted_events
             BEFORE INSERT ON event_headers
             FOR EACH ROW
             BEGIN
-                -- Checks if this event hasn't already been deleted. If so, reject it.
-
-                -- Highly optimized using hash + created_at (duplicated in tags) index:
-                ---- Expects tag_hash to be non-existant for new events (quick exit)
-                ---- Expects created-at >= event's to not exist (quick exit)
                 SELECT RAISE(ABORT, 'blocked: a deletion event exists')
                 WHERE EXISTS (
-                    SELECT 1 FROM event_tags
+                    $sql
-                    WHERE
-                        event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND
-                        event_tags.kind = 5 AND
-                        event_tags.pubkey_hash = NEW.pubkey_owner_hash AND
-                        event_tags.created_at >= NEW.created_at
                 );
             END;
             """.trimIndent(),

quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt

@@ -75,8 +75,9 @@ class EventIndexesModule(
             }
 
         // queries by limit (latest records), since, until (sync all) alone without any filter by kind.. rare
-        // serves as fall back for the lack of query_by_tags_hash index by default
+        if (indexStrategy.indexEventsByCreatedAtAlone) {
-        db.execSQL("CREATE INDEX query_by_created_at_id        ON event_headers ($orderBy)")
+            db.execSQL("CREATE INDEX query_by_created_at_id        ON event_headers ($orderBy)")
+        }
 
         // queries by kind only, mostly used in Global Feeds when author is not important.
         db.execSQL("CREATE INDEX query_by_kind_created         ON event_headers (kind, $orderBy)")
@@ -87,10 +88,10 @@ class EventIndexesModule(
         // makes deletions on the event_header fast
         db.execSQL("CREATE INDEX fk_event_tags_header_id       ON event_tags (event_header_row_id)")
 
-        // ---------------------------------------------------------------
+        // ---------------------------------------------------------------------------
-        // This are a very slow indexes (80% of the insert time goes here)
+        // These next 3 are a very slow indexes (80% of the insert time goes here)
-        // ---------------------------------------------------------------
+        // ---------------------------------------------------------------------------
-        if (indexStrategy.indexTagQueriesWithoutKinds) {
+        if (indexStrategy.indexTagsByCreatedAtAlone) {
             // First one is only needed if the user is searching by tags without a kind.
             db.execSQL("CREATE INDEX query_by_tags_hash          ON event_tags (tag_hash, created_at DESC)")
         }
@@ -98,8 +99,10 @@ class EventIndexesModule(
         // This is the default index for most clients: tags by specific kinds that are supported by the client.
         db.execSQL("CREATE INDEX query_by_tags_hash_kind         ON event_tags (tag_hash, kind, created_at DESC)")
 
-        // this one is to allow search of tags by kind and author at the same time: DMs, reports,
+        // this one is to allow search of tags by kind and author at the same time: NIP-04 DMs, reports,
-        db.execSQL("CREATE INDEX query_by_tags_hash_kind_pubkey  ON event_tags (tag_hash, kind, pubkey_hash, created_at DESC)")
+        if (indexStrategy.indexTagsWithKindAndPubkey) {
+            db.execSQL("CREATE INDEX query_by_tags_hash_kind_pubkey  ON event_tags (tag_hash, kind, pubkey_hash, created_at DESC)")
+        }
 
         // Prevent updates to maintain immutability
         db.execSQL(

quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt

@@ -23,7 +23,62 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite
 import com.vitorpamplona.quartz.nip01Core.core.Tag
 
 interface IndexingStrategy {
-    val indexTagQueriesWithoutKinds: Boolean
+    /**
+     * Activate this if you see too many Filters with just LIMIT, SINCE and
+     * UNTIL filled up.
+     *
+     * Clients never support all kinds, so this is usually
+     * only done with syncing services that must download ALL kinds from
+     * ALL authors.
+     *
+     * The index will make these queries significantly faster, but maybe speed
+     * is not a requirement on Sync services. The size of this index is
+     * considerable.
+     *
+     * Keep in mind that activating too many indexes increases the size of the
+     * DB so much that the indexes themselves won't fit in memory, requiring
+     * frequent reloadings of the index itself from disk.
+     */
+    val indexEventsByCreatedAtAlone: Boolean
+
+    /**
+     * Activate this if you see too many Tag-centric Filters without
+     * kind, pubkey or id.
+     *
+     * Clients never support all kinds, so this is usually
+     * only done in rare usecases where the client supports all
+     * kinds.
+     *
+     * The index will make these queries significantly faster, but maybe speed
+     * is not a requirement on such services. Because this is an index in
+     * event tags, it becomes QUITE BIG.
+     *
+     * Keep in mind that activating too many indexes increases the size of the
+     * DB so much that the indexes themselves won't fit in memory, requiring
+     * frequent reloadings of the index itself from disk.
+     */
+    val indexTagsByCreatedAtAlone: Boolean
+
+    /**
+     * Activate this if you see too many Tag-centric Filters without
+     * kind AND pubkey at the same time.
+     *
+     * This is a rarely used index (reports by your follows or
+     * NIP-04 DMs for instance) that becomes quite large without
+     * major gains.
+     *
+     * Keep in mind that activating too many indexes increases the size of the
+     * DB so much that the indexes themselves won't fit in memory, requiring
+     * frequent reloadings of the index itself from disk.
+     */
+    val indexTagsWithKindAndPubkey: Boolean
+
+    /**
+     * Activate this to make sure queries are always in order when
+     * the same created_at exists. This will impact performance and
+     * the size of indexes, but it provides results that are compliant
+     * with the Nostr Spec
+     */
     val useAndIndexIdOnOrderBy: Boolean
 
     fun shouldIndex(
@@ -36,7 +91,9 @@ interface IndexingStrategy {
  * By default, we index all tags that have a single letter name and some value
  */
 class DefaultIndexingStrategy(
-    override val indexTagQueriesWithoutKinds: Boolean = false,
+    override val indexEventsByCreatedAtAlone: Boolean = false,
+    override val indexTagsByCreatedAtAlone: Boolean = false,
+    override val indexTagsWithKindAndPubkey: Boolean = false,
     override val useAndIndexIdOnOrderBy: Boolean = false,
 ) : IndexingStrategy {
     override fun shouldIndex(

quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt

@@ -655,7 +655,7 @@ class QueryBuilder(
                 if (project) {
                     append("\nORDER BY created_at DESC")
                     if (indexStrategy.useAndIndexIdOnOrderBy) {
-                        append(", event_headers.id ASC")
+                        append(", id ASC")
                     }
                 }
                 if (limit != null) {