nick/amethyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Adds more settings to pick different ways to build the SQL database

Browse Source
This commit is contained in:
Vitor Pamplona
2026-01-02 14:54:30 -05:00
parent 7c63d654f2
commit 307cff87e7
19 changed files with 1452 additions and 1285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt
+27 -7
View File
@@ -29,7 +29,7 @@ import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
class EventIndexesModule(
val hasher: (db: SQLiteDatabase) -> TagNameValueHasher,
val tagIndexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
) : IModule {
override fun create(db: SQLiteDatabase) {
db.execSQL(
@@ -67,18 +67,38 @@ class EventIndexesModule(
// queries by ID (load events)
db.execSQL("CREATE UNIQUE INDEX event_headers_id ON event_headers (id)")
// queries by limit (latest records), since, until (sync all) alone
db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers (created_at DESC, id)")
val orderBy =
if (indexStrategy.useAndIndexIdOnOrderBy) {
"created_at DESC, id ASC"
} else {
"created_at DESC"
}
// queries by limit (latest records), since, until (sync all) alone without any filter by kind.. rare
// serves as fall back for the lack of query_by_tags_hash index by default
db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers ($orderBy)")
// queries by kind only, mostly used in Global Feeds when author is not important.
db.execSQL("CREATE INDEX query_by_kind_created ON event_headers (kind, $orderBy)")
// queries by kind + pubkey, but not d-tag, even if they are replaceables and addressables, by date.
db.execSQL("CREATE INDEX query_by_kind_pubkey_created ON event_headers (kind, pubkey, created_at DESC)")
db.execSQL("CREATE INDEX query_by_kind_pubkey_created ON event_headers (kind, pubkey, $orderBy)")
// makes deletions on the event_header fast
db.execSQL("CREATE INDEX fk_event_tags_header_id ON event_tags (event_header_row_id)")
// This is a very slow index to build (80% of the insert time goes here) but it is extremely effective.
db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, created_at DESC)")
// ---------------------------------------------------------------
// This are a very slow indexes (80% of the insert time goes here)
// ---------------------------------------------------------------
if (indexStrategy.indexTagQueriesWithoutKinds) {
// First one is only needed if the user is searching by tags without a kind.
db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, created_at DESC)")
}
// This is the default index for most clients: tags by specific kinds that are supported by the client.
db.execSQL("CREATE INDEX query_by_tags_hash_kind ON event_tags (tag_hash, kind, created_at DESC)")
// this one is to allow search of tags by kind and author at the same time: DMs, reports,
db.execSQL("CREATE INDEX query_by_tags_hash_kind_pubkey ON event_tags (tag_hash, kind, pubkey_hash, created_at DESC)")
// Prevent updates to maintain immutability
@@ -173,7 +193,7 @@ class EventIndexesModule(
// rebalancing the tree every new insert
val indexableTags = ArrayList<Long>()
for (idx in event.tags.indices) {
if (tagIndexStrategy.shouldIndex(event.kind, event.tags[idx])) {
if (indexStrategy.shouldIndex(event.kind, event.tags[idx])) {
indexableTags.add(hasher.hash(event.tags[idx][0], event.tags[idx][1]))
}
}
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventStore.kt
+2 -2
View File
@@ -29,9 +29,9 @@ class EventStore(
context: Context,
dbName: String? = "events.db",
val relayUrl: String? = "wss://quartz.local",
val tagIndexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
) : IEventStore {
val store = SQLiteEventStore(context, dbName, relayUrl, tagIndexStrategy)
val store = SQLiteEventStore(context, dbName, relayUrl, indexStrategy)
override fun insert(event: Event) = store.insertEvent(event)
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt
+7 -1
View File
@@ -23,6 +23,9 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite
import com.vitorpamplona.quartz.nip01Core.core.Tag
interface IndexingStrategy {
val indexTagQueriesWithoutKinds: Boolean
val useAndIndexIdOnOrderBy: Boolean
fun shouldIndex(
kind: Int,
tag: Tag,
@@ -32,7 +35,10 @@ interface IndexingStrategy {
/**
* By default, we index all tags that have a single letter name and some value
*/
class DefaultIndexingStrategy : IndexingStrategy {
class DefaultIndexingStrategy(
override val indexTagQueriesWithoutKinds: Boolean = false,
override val useAndIndexIdOnOrderBy: Boolean = false,
) : IndexingStrategy {
override fun shouldIndex(
kind: Int,
tag: Tag,
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt
+214 -24
View File
@@ -23,6 +23,8 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite
import android.database.Cursor
import android.database.sqlite.SQLiteDatabase
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.core.Kind
import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper
import com.vitorpamplona.quartz.nip01Core.core.isAddressable
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
@@ -34,6 +36,7 @@ import kotlin.collections.component2
class QueryBuilder(
val fts: FullTextSearchModule,
val hasher: (db: SQLiteDatabase) -> TagNameValueHasher,
val indexStrategy: IndexingStrategy,
) {
// ------------
// Main methods
@@ -110,13 +113,37 @@ class QueryBuilder(
filter: Filter,
hasher: TagNameValueHasher,
): QuerySpec {
val newFilter = filter.toFilterWithDTags()
if (newFilter.isSimpleQuery()) {
return makeSimpleQuery(
ids = newFilter.ids,
authors = newFilter.authors,
kinds = newFilter.kinds,
dTags = newFilter.dTags,
since = newFilter.since,
until = newFilter.until,
limit = newFilter.limit,
)
}
if (newFilter.isSimpleSearch()) {
return makeSimpleSearch(
search = newFilter.search!!,
ids = newFilter.ids,
authors = newFilter.authors,
kinds = newFilter.kinds,
dTags = newFilter.dTags,
since = newFilter.since,
until = newFilter.until,
limit = newFilter.limit,
)
}
val rowIdSubqueries = prepareRowIDSubQueries(filter, hasher)
return if (rowIdSubqueries == null) {
QuerySpec(
makeEverythingQuery(),
emptyList(),
)
QuerySpec(makeEverythingQuery())
} else {
QuerySpec(
makeQueryIn(rowIdSubqueries.sql),
@@ -129,6 +156,8 @@ class QueryBuilder(
filters: List<Filter>,
hasher: TagNameValueHasher,
): QuerySpec {
if (filters.size == 1) return toSql(filters.first(), hasher)
val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher)
return if (rowIdSubqueries == null) {
@@ -144,7 +173,7 @@ class QueryBuilder(
}
}
private fun makeEverythingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC, id"
private fun makeEverythingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC${if (indexStrategy.useAndIndexIdOnOrderBy) ", id ASC" else ""}"
private fun makeQueryIn(rowIdQuery: String) =
"""
@@ -153,7 +182,7 @@ class QueryBuilder(
$rowIdQuery
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY created_at DESC, id
ORDER BY created_at DESC${if (indexStrategy.useAndIndexIdOnOrderBy) ", id ASC" else ""}
""".trimIndent()
private fun <T : Event> SQLiteDatabase.runQuery(query: QuerySpec): List<T> =
@@ -359,12 +388,12 @@ class QueryBuilder(
buildString {
// always do tags if there are any
if (reverseLookup) {
append("SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags ")
append("SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags")
// it's quite rare to have 2 tags in the filter, but possible
nonDTagsIn.keys.forEachIndexed { index, tagName ->
if (defaultTagKey != null) {
append("INNER JOIN event_tags as event_tagsIn$index ON event_tagsIn$index.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn$index.created_at = event_tags.created_at ")
append(" INNER JOIN event_tags as event_tagsIn$index ON event_tagsIn$index.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn$index.created_at = event_tags.created_at")
} else {
defaultTagKey = TagNameForQuery.InTags(tagName)
}
@@ -373,7 +402,7 @@ class QueryBuilder(
nonDTagsAll.keys.forEachIndexed { index, tagName ->
nonDTagsAll[tagName]!!.forEachIndexed { valueIndex, tagValue ->
if (defaultTagKey != null) {
append("INNER JOIN event_tags as event_tagsAll${index}_$valueIndex ON event_tagsAll${index}_$valueIndex.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll${index}_$valueIndex.created_at = event_tags.created_at ")
append(" INNER JOIN event_tags as event_tagsAll${index}_$valueIndex ON event_tagsAll${index}_$valueIndex.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll${index}_$valueIndex.created_at = event_tags.created_at")
} else {
defaultTagKey = TagNameForQuery.AllTags(tagName, valueIndex)
}
@@ -381,21 +410,21 @@ class QueryBuilder(
}
if (needHeaders) {
append("INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id ")
append(" INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id")
}
if (mustJoinSearch) {
append("INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id ")
append(" INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id")
}
} else if (mustJoinSearch) {
append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName} ")
append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName}")
if (hasHeaders) {
append("INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
append(" INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
}
} else {
// no tags and no search.
append("SELECT event_headers.row_id as row_id FROM event_headers ")
append("SELECT event_headers.row_id as row_id FROM event_headers")
}
}
@@ -480,22 +509,183 @@ class QueryBuilder(
}
}
val whereClause =
if (filter.limit != null) {
if (reverseLookup) {
"${clause.conditions} ORDER BY event_tags.created_at DESC LIMIT ${filter.limit}"
} else {
"${clause.conditions} ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT ${filter.limit}"
val sql =
buildString {
append(projection)
if (clause.conditions.isNotEmpty()) {
append(" WHERE ${clause.conditions}")
}
if (filter.limit != null) {
if (reverseLookup) {
append(" ORDER BY event_tags.created_at DESC")
append(" LIMIT ")
append(filter.limit)
} else {
append(" ORDER BY event_headers.created_at DESC")
append(" LIMIT ")
append(filter.limit)
}
}
} else {
clause.conditions
}
return QuerySpec("$projection WHERE $whereClause", clause.args)
return QuerySpec(sql, clause.args)
}
private fun makeSimpleSearch(
search: String,
ids: List<HexKey>? = null,
authors: List<HexKey>? = null,
kinds: List<Kind>? = null,
dTags: List<String>? = null,
since: Long? = null,
until: Long? = null,
limit: Int? = null,
): QuerySpec {
val clause =
where {
// the order should match indexes
// ids reduce the filter the most
ids?.let { equalsOrIn("event_headers.id", it) }
match(fts.tableName, search)
kinds?.let { equalsOrIn("event_headers.kind", it) }
authors?.let { equalsOrIn("event_headers.pubkey", it) }
// there are indexes for these, starting with tags.
dTags?.let { equalsOrIn("event_headers.d_tag", it) }
since?.let { greaterThanOrEquals("event_headers.created_at", it) }
until?.let { lessThanOrEquals("event_headers.created_at", it) }
// if this is a dTag filter, it is likely that all kinds are addressables
// and so force the use of the addressable index
if (dTags != null && kinds != null) {
if (kinds.all { it.isAddressable() }) {
// matches unique index kind >= 30000 AND kind < 40000
raw("(event_headers.kind >= 30000 AND kind < 40000)")
}
}
}
val sql =
buildString {
append("SELECT event_headers.id, event_headers.pubkey, event_headers.created_at, event_headers.kind, event_headers.tags, event_headers.content, event_headers.sig FROM event_headers")
append("\nINNER JOIN ${fts.tableName} ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}")
if (clause.conditions.isNotEmpty()) {
append("\nWHERE ${clause.conditions}")
}
append("\nORDER BY event_headers.created_at DESC")
if (indexStrategy.useAndIndexIdOnOrderBy) {
append(", event_headers.id ASC")
}
if (limit != null) {
append("\nLIMIT ")
append(limit)
}
}
println(sql)
return QuerySpec(sql, clause.args)
}
private fun makeSimpleQuery(
ids: List<HexKey>? = null,
authors: List<HexKey>? = null,
kinds: List<Kind>? = null,
dTags: List<String>? = null,
since: Long? = null,
until: Long? = null,
limit: Int? = null,
): QuerySpec {
val clause =
where {
// the order should match indexes
// ids reduce the filter the most
ids?.let { equalsOrIn("id", it) }
kinds?.let { equalsOrIn("kind", it) }
authors?.let { equalsOrIn("pubkey", it) }
// there are indexes for these, starting with tags.
dTags?.let { equalsOrIn("d_tag", it) }
since?.let { greaterThanOrEquals("created_at", it) }
until?.let { lessThanOrEquals("created_at", it) }
// if this is a dTag filter, it is likely that all kinds are addressables
// and so force the use of the addressable index
if (dTags != null && kinds != null) {
if (kinds.all { it.isAddressable() }) {
// matches unique index kind >= 30000 AND kind < 40000
raw("(kind >= 30000 AND kind < 40000)")
}
}
}
val sql =
buildString {
append("SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers")
if (clause.conditions.isNotEmpty()) {
append("\nWHERE ")
append(clause.conditions)
}
append("\nORDER BY created_at DESC")
if (indexStrategy.useAndIndexIdOnOrderBy) {
append(", event_headers.id ASC")
}
if (limit != null) {
append("\nLIMIT ")
append(limit)
}
}
println(sql)
return QuerySpec(sql, clause.args)
}
class FilterWithDTags(
val ids: List<HexKey>? = null,
val authors: List<HexKey>? = null,
val kinds: List<Kind>? = null,
val dTags: List<String>? = null,
val nonDTagsIn: Map<String, List<String>>? = null,
val nonDTagsAll: Map<String, List<String>>? = null,
val since: Long? = null,
val until: Long? = null,
val limit: Int? = null,
val search: String? = null,
) {
fun isSimpleSearch() =
search != null && search.isNotEmpty() &&
(nonDTagsIn == null || nonDTagsIn.isEmpty()) &&
(nonDTagsAll == null || nonDTagsAll.isEmpty())
// can be resolved with just event_headers
fun isSimpleQuery() =
(nonDTagsIn == null || nonDTagsIn.isEmpty()) &&
(nonDTagsAll == null || nonDTagsAll.isEmpty()) &&
(search == null || search.isEmpty())
}
fun Filter.toFilterWithDTags(): FilterWithDTags =
FilterWithDTags(
ids = ids,
authors = authors,
kinds = kinds,
dTags = tags?.get("d") ?: tagsAll?.get("d"),
nonDTagsIn = tags?.filter { it.key != "d" }?.ifEmpty { null },
nonDTagsAll = tagsAll?.filter { it.key != "d" }?.ifEmpty { null },
since = since,
until = until,
limit = limit,
search = search,
)
data class QuerySpec(
val sql: String,
val args: List<String>,
val args: List<String> = emptyList(),
)
}
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt
+3 -3
View File
@@ -41,7 +41,7 @@ class SQLiteEventStore(
val context: Context,
val dbName: String? = "events.db",
val relayUrl: String? = null,
val tagIndexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
val indexStrategy: IndexingStrategy = DefaultIndexingStrategy(),
) : SQLiteOpenHelper(context, dbName, null, DATABASE_VERSION) {
companion object {
const val DATABASE_VERSION = 2
@@ -50,7 +50,7 @@ class SQLiteEventStore(
val seedModule = SeedModule()
val fullTextSearchModule = FullTextSearchModule()
val eventIndexModule = EventIndexesModule(seedModule::hasher, tagIndexStrategy)
val eventIndexModule = EventIndexesModule(seedModule::hasher, indexStrategy)
val replaceableModule = ReplaceableModule()
val addressableModule = AddressableModule()
@@ -60,7 +60,7 @@ class SQLiteEventStore(
val expirationModule = ExpirationModule()
val rightToVanishModule = RightToVanishModule(seedModule::hasher)
val queryBuilder = QueryBuilder(fullTextSearchModule, seedModule::hasher)
val queryBuilder = QueryBuilder(fullTextSearchModule, seedModule::hasher, indexStrategy)
val modules =
listOf(
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/sql/Condition.kt
+2
View File
@@ -85,4 +85,6 @@ sealed class Condition {
data class Or(
val conditions: List<Condition>,
) : Condition()
class Empty : Condition()
}
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/sql/SqlSelectionBuilder.kt
+3
View File
@@ -38,6 +38,9 @@ class SqlSelectionBuilder(
*/
private fun buildCondition(cond: Condition): String =
when (cond) {
is Condition.Empty -> {
""
}
is Condition.Raw -> {
cond.condition
}
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/sql/WhereClauseBuilder.kt
+1 -1
View File
@@ -119,7 +119,7 @@ class WhereClauseBuilder {
}
fun where(block: WhereClauseBuilder.() -> Unit): WhereClause {
val condition = WhereClauseBuilder().apply(block).buildAnd() ?: Condition.And(emptyList())
val condition = WhereClauseBuilder().apply(block).buildAnd() ?: Condition.Empty()
return SqlSelectionBuilder(condition).build()
}