nick/amethyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Creating additional configuration options and amplifying tests to a combination of all those options

Browse Source
This commit is contained in:
Vitor Pamplona
2026-01-03 18:18:36 -05:00
parent 98ccb5e0ae
commit d7888c57fa
7 changed files with 1105 additions and 675 deletions
Download Patch File Download Diff File Expand all files Collapse all files
quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/BaseDBTest.kt
+35 -8
View File
@@ -26,19 +26,46 @@ import org.junit.After
import org.junit.Before
open class BaseDBTest {
private lateinit var dbs: Map<String, EventStore>
private lateinit var dbs: MutableMap<String, EventStore>
fun DefaultIndexingStrategy.name(): String =
"""
indexEventsByCreatedAtAlone=$indexEventsByCreatedAtAlone
indexTagsByCreatedAtAlone=$indexTagsByCreatedAtAlone
indexTagsWithKindAndPubkey=$indexTagsWithKindAndPubkey
useAndIndexIdOnOrderBy=$useAndIndexIdOnOrderBy
""".trimIndent()
@Before
fun setup() {
val context = ApplicationProvider.getApplicationContext<Context>()
dbs =
mapOf(
"Default" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(false, false)),
"IndexAll" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(true, false)),
"Order by ID" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(false, true)),
"IndexAll, Order by ID" to EventStore(context, null, indexStrategy = DefaultIndexingStrategy(true, true)),
)
val booleans = listOf(true, false)
dbs = mutableMapOf<String, EventStore>()
// tests all possible DBs
for (indexEventsByCreatedAtAlone in booleans) {
for (indexTagsByCreatedAtAlone in booleans) {
for (indexTagsWithKindAndPubkey in booleans) {
for (useAndIndexIdOnOrderBy in booleans) {
val indexStrategy =
DefaultIndexingStrategy(
indexEventsByCreatedAtAlone,
indexTagsByCreatedAtAlone,
indexTagsWithKindAndPubkey,
useAndIndexIdOnOrderBy,
)
dbs[indexStrategy.name()] =
EventStore(
context = context,
dbName = null,
indexStrategy = indexStrategy,
)
}
}
}
}
}
@After
quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt
+24 -18
View File
@@ -208,26 +208,32 @@ class DeletionTest : BaseDBTest() {
@Test
fun testTriggersIndexUsage() =
forEachDB { db ->
val sql =
"""
SELECT 1 FROM event_tags
WHERE
event_tags.tag_hash IN (3221122, 223322) AND
event_tags.kind = 5 AND
event_tags.pubkey_hash = 22332323 AND
event_tags.created_at >= 1766686500
""".trimIndent()
var sql = db.store.deletionModule.rejectDeletedEventsSQLTemplate()
val explainer =
db.store.explainQuery(sql)
sql = sql.replace("NEW.etag_hash", "3221122")
sql = sql.replace("NEW.atag_hash", "223322")
sql = sql.replace("NEW.pubkey_owner_hash", "22332323")
sql = sql.replace("NEW.created_at", "1766686500")
TestCase.assertEquals(
"""
${sql.replace("\n","\n ")}
└── SEARCH event_tags USING COVERING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?)
""".trimIndent(),
explainer,
)
val explainer = db.store.explainQuery(sql)
if (db.indexStrategy.indexTagsWithKindAndPubkey) {
TestCase.assertEquals(
"""
|$sql
|└── SEARCH event_tags USING COVERING INDEX query_by_tags_hash_kind_pubkey (tag_hash=? AND kind=? AND pubkey_hash=? AND created_at>?)
""".trimMargin(),
explainer,
)
} else {
TestCase.assertEquals(
"""
|$sql
|└── SEARCH event_tags USING INDEX query_by_tags_hash_kind (tag_hash=? AND kind=? AND created_at>?)
""".trimMargin(),
explainer,
)
}
}
@Test
quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt
+951 -627
View File
File diff suppressed because it is too large Load Diff
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt
+24 -11
View File
@@ -30,32 +30,45 @@ import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent
class DeletionRequestModule(
val hasher: (db: SQLiteDatabase) -> TagNameValueHasher,
val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
) : IModule {
fun rejectDeletedEventsSQLTemplate(): String =
if (indexStrategy.indexTagsWithKindAndPubkey) {
"""
|SELECT 1 FROM event_tags
|WHERE
| event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND
| event_tags.kind = 5 AND
| event_tags.pubkey_hash = NEW.pubkey_owner_hash AND
| event_tags.created_at >= NEW.created_at
""".trimMargin()
} else {
"""
|SELECT 1 FROM event_tags
|WHERE
| event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND
| event_tags.kind = 5 AND
| event_tags.created_at >= NEW.created_at AND
| event_tags.pubkey_hash = NEW.pubkey_owner_hash
""".trimMargin()
}
/**
* Creates a trigger to reject events that have been
* deleted by ID or ATag including GiftWraps that
* must be checked against the p-tag (pubkey_owner_hash)
*/
override fun create(db: SQLiteDatabase) {
val sql = rejectDeletedEventsSQLTemplate().replace("\n", "\n ")
db.execSQL(
"""
CREATE TRIGGER reject_deleted_events
BEFORE INSERT ON event_headers
FOR EACH ROW
BEGIN
-- Checks if this event hasn't already been deleted. If so, reject it.
-- Highly optimized using hash + created_at (duplicated in tags) index:
---- Expects tag_hash to be non-existant for new events (quick exit)
---- Expects created-at >= event's to not exist (quick exit)
SELECT RAISE(ABORT, 'blocked: a deletion event exists')
WHERE EXISTS (
SELECT 1 FROM event_tags
WHERE
event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND
event_tags.kind = 5 AND
event_tags.pubkey_hash = NEW.pubkey_owner_hash AND
event_tags.created_at >= NEW.created_at
$sql
);
END;
""".trimIndent(),
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt
+11 -8
View File
@@ -75,8 +75,9 @@ class EventIndexesModule(
}
// queries by limit (latest records), since, until (sync all) alone without any filter by kind.. rare
// serves as fall back for the lack of query_by_tags_hash index by default
db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers ($orderBy)")
if (indexStrategy.indexEventsByCreatedAtAlone) {
db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers ($orderBy)")
}
// queries by kind only, mostly used in Global Feeds when author is not important.
db.execSQL("CREATE INDEX query_by_kind_created ON event_headers (kind, $orderBy)")
@@ -87,10 +88,10 @@ class EventIndexesModule(
// makes deletions on the event_header fast
db.execSQL("CREATE INDEX fk_event_tags_header_id ON event_tags (event_header_row_id)")
// ---------------------------------------------------------------
// This are a very slow indexes (80% of the insert time goes here)
// ---------------------------------------------------------------
if (indexStrategy.indexTagQueriesWithoutKinds) {
// ---------------------------------------------------------------------------
// These next 3 are a very slow indexes (80% of the insert time goes here)
// ---------------------------------------------------------------------------
if (indexStrategy.indexTagsByCreatedAtAlone) {
// First one is only needed if the user is searching by tags without a kind.
db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, created_at DESC)")
}
@@ -98,8 +99,10 @@ class EventIndexesModule(
// This is the default index for most clients: tags by specific kinds that are supported by the client.
db.execSQL("CREATE INDEX query_by_tags_hash_kind ON event_tags (tag_hash, kind, created_at DESC)")
// this one is to allow search of tags by kind and author at the same time: DMs, reports,
db.execSQL("CREATE INDEX query_by_tags_hash_kind_pubkey ON event_tags (tag_hash, kind, pubkey_hash, created_at DESC)")
// this one is to allow search of tags by kind and author at the same time: NIP-04 DMs, reports,
if (indexStrategy.indexTagsWithKindAndPubkey) {
db.execSQL("CREATE INDEX query_by_tags_hash_kind_pubkey ON event_tags (tag_hash, kind, pubkey_hash, created_at DESC)")
}
// Prevent updates to maintain immutability
db.execSQL(
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt
+59 -2
View File
@@ -23,7 +23,62 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite
import com.vitorpamplona.quartz.nip01Core.core.Tag
interface IndexingStrategy {
val indexTagQueriesWithoutKinds: Boolean
/**
* Activate this if you see too many Filters with just LIMIT, SINCE and
* UNTIL filled up.
*
* Clients never support all kinds, so this is usually
* only done with syncing services that must download ALL kinds from
* ALL authors.
*
* The index will make these queries significantly faster, but maybe speed
* is not a requirement on Sync services. The size of this index is
* considerable.
*
* Keep in mind that activating too many indexes increases the size of the
* DB so much that the indexes themselves won't fit in memory, requiring
* frequent reloadings of the index itself from disk.
*/
val indexEventsByCreatedAtAlone: Boolean
/**
* Activate this if you see too many Tag-centric Filters without
* kind, pubkey or id.
*
* Clients never support all kinds, so this is usually
* only done in rare usecases where the client supports all
* kinds.
*
* The index will make these queries significantly faster, but maybe speed
* is not a requirement on such services. Because this is an index in
* event tags, it becomes QUITE BIG.
*
* Keep in mind that activating too many indexes increases the size of the
* DB so much that the indexes themselves won't fit in memory, requiring
* frequent reloadings of the index itself from disk.
*/
val indexTagsByCreatedAtAlone: Boolean
/**
* Activate this if you see too many Tag-centric Filters without
* kind AND pubkey at the same time.
*
* This is a rarely used index (reports by your follows or
* NIP-04 DMs for instance) that becomes quite large without
* major gains.
*
* Keep in mind that activating too many indexes increases the size of the
* DB so much that the indexes themselves won't fit in memory, requiring
* frequent reloadings of the index itself from disk.
*/
val indexTagsWithKindAndPubkey: Boolean
/**
* Activate this to make sure queries are always in order when
* the same created_at exists. This will impact performance and
* the size of indexes, but it provides results that are compliant
* with the Nostr Spec
*/
val useAndIndexIdOnOrderBy: Boolean
fun shouldIndex(
@@ -36,7 +91,9 @@ interface IndexingStrategy {
* By default, we index all tags that have a single letter name and some value
*/
class DefaultIndexingStrategy(
override val indexTagQueriesWithoutKinds: Boolean = false,
override val indexEventsByCreatedAtAlone: Boolean = false,
override val indexTagsByCreatedAtAlone: Boolean = false,
override val indexTagsWithKindAndPubkey: Boolean = false,
override val useAndIndexIdOnOrderBy: Boolean = false,
) : IndexingStrategy {
override fun shouldIndex(
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt
+1 -1
View File
@@ -655,7 +655,7 @@ class QueryBuilder(
if (project) {
append("\nORDER BY created_at DESC")
if (indexStrategy.useAndIndexIdOnOrderBy) {
append(", event_headers.id ASC")
append(", id ASC")
}
}
if (limit != null) {