diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt index 9a6000835..67162ad12 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -32,7 +32,7 @@ import org.junit.Test class QueryAssemblerTest { val hasher = TagNameValueHasher(0) - val builder = EventIndexesModule(FullTextSearchModule(), { hasher }) + val builder = QueryBuilder(FullTextSearchModule(), { hasher }) val key1 = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" val key2 = "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14" diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt index 981260c58..e8410d5a5 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt @@ -20,18 +20,12 @@ */ package com.vitorpamplona.quartz.nip01Core.store.sqlite -import android.database.Cursor import android.database.sqlite.SQLiteDatabase import com.vitorpamplona.quartz.nip01Core.core.AddressSerializer import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip01Core.core.HexKey -import com.vitorpamplona.quartz.nip01Core.core.Kind import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import com.vitorpamplona.quartz.nip01Core.store.sqlite.sql.where import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent -import com.vitorpamplona.quartz.utils.EventFactory class EventIndexesModule( val fts: FullTextSearchModule, @@ -188,437 +182,8 @@ class EventIndexesModule( return headerId } - fun planQuery( - filter: Filter, - hasher: TagNameValueHasher, - db: SQLiteDatabase, - ): String { - val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher) - - return if (rowIdSubQuery == null) { - val query = makeEverythingQuery() - db.explainQuery(query) - } else { - val query = makeQueryIn(rowIdSubQuery.sql) - db.explainQuery(query, rowIdSubQuery.args.toTypedArray()) - } - } - - fun query( - filter: Filter, - db: SQLiteDatabase, - ): List { - val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher(db)) - - return if (rowIdSubQuery == null) { - db.runQuery(makeEverythingQuery()) - } else { - db.runQuery(makeQueryIn(rowIdSubQuery.sql), rowIdSubQuery.args) - } - } - - fun query( - filter: Filter, - db: SQLiteDatabase, - onEach: (T) -> Unit, - ) { - val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher(db)) - - return if (rowIdSubQuery == null) { - db.runQuery(makeEverythingQuery(), onEach = onEach) - } else { - db.runQuery(makeQueryIn(rowIdSubQuery.sql), rowIdSubQuery.args, onEach) - } - } - - fun planQuery( - filters: List, - hasher: TagNameValueHasher, - db: SQLiteDatabase, - ): String { - val rowIdSubQuery = unionSubqueriesIfNeeded(filters, hasher) - - return if (rowIdSubQuery == null) { - val query = makeEverythingQuery() - db.explainQuery(query) - } else { - val query = makeQueryIn(rowIdSubQuery.sql) - db.explainQuery(query, rowIdSubQuery.args.toTypedArray()) - } - } - - fun query( - filters: List, - db: SQLiteDatabase, - ): List { - val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return db.runQuery(makeEverythingQuery()) - return db.runQuery(makeQueryIn(rowIdSubqueries.sql), rowIdSubqueries.args) - } - - fun query( - filters: List, - db: SQLiteDatabase, - onEach: (T) -> Unit, - ) { - val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) - - if (rowIdSubqueries == null) { - db.runQuery(makeEverythingQuery(), onEach = onEach) - } else { - db.runQuery(makeQueryIn(rowIdSubqueries.sql), rowIdSubqueries.args, onEach) - } - } - - private fun makeEverythingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC, id" - - private fun makeQueryIn(rowIdQuery: String) = - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ( - $rowIdQuery - ) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC, id - """.trimIndent() - - private fun SQLiteDatabase.runQuery( - sql: String, - args: List = emptyList(), - ): List = - rawQuery(sql, args.toTypedArray()).use { cursor -> - ArrayList(cursor.count).apply { - while (cursor.moveToNext()) { - add(cursor.toEvent()) - } - } - } - - private inline fun SQLiteDatabase.runQuery( - sql: String, - args: List = emptyList(), - onEach: (T) -> Unit, - ) = rawQuery(sql, args.toTypedArray()).use { cursor -> - while (cursor.moveToNext()) { - onEach(cursor.toEvent()) - } - } - - private fun Cursor.toEvent() = - EventFactory.create( - getString(0).intern(), - getString(1).intern(), - getLong(2), - getInt(3), - OptimizedJsonMapper.fromJsonToTagArray(getString(4)), - getString(5), - getString(6), - ) - - class RawEvent( - val id: HexKey, - val pubKey: HexKey, - val createdAt: Long, - val kind: Kind, - val jsonTags: String, - val content: String, - val sig: HexKey, - ) { - fun toEvent() = - EventFactory.create( - id.intern(), - pubKey.intern(), - createdAt, - kind, - OptimizedJsonMapper.fromJsonToTagArray(jsonTags), - content, - sig, - ) - } - - private fun Cursor.toRawEvent() = - RawEvent( - getString(0), - getString(1), - getLong(2), - getInt(3), - getString(4), - getString(5), - getString(6), - ) - - // -------------- - // Counts - // ------------- - fun count( - filter: Filter, - db: SQLiteDatabase, - ): Int { - val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher(db)) - - return if (rowIdSubQuery == null) { - db.countEverything() - } else { - db.countIn(rowIdSubQuery.sql, rowIdSubQuery.args) - } - } - - fun count( - filters: List, - db: SQLiteDatabase, - ): Int { - val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return db.countEverything() - - return db.countIn(rowIdSubqueries.sql, rowIdSubqueries.args) - } - - private fun SQLiteDatabase.countEverything() = runCount("SELECT count(*) as count FROM event_headers") - - private fun SQLiteDatabase.countIn( - rowIdQuery: String, - args: List, - ) = runCount("SELECT COUNT(*) as count FROM ($rowIdQuery)", args) - - private fun SQLiteDatabase.runCount( - sql: String, - args: List = emptyList(), - ): Int = - rawQuery(sql, args.toTypedArray()).use { cursor -> - cursor.moveToNext() - cursor.getInt(0) - } - - // -------------- - // Deletes - // ------------- - fun delete( - filter: Filter, - db: SQLiteDatabase, - ): Int { - val rowIdQuery = prepareRowIDSubQueries(filter, hasher(db)) - - return if (rowIdQuery == null) { - 0 - } else { - db.runDelete(rowIdQuery.sql, rowIdQuery.args) - } - } - - fun delete( - filters: List, - db: SQLiteDatabase, - ): Int { - val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return 0 - - return db.runDelete(rowIdSubqueries.sql, rowIdSubqueries.args) - } - - private fun SQLiteDatabase.runDelete( - sql: String, - args: List = emptyList(), - ): Int = delete("event_headers", "row_id IN ($sql)", args.toTypedArray()) - - // --------------------------------- - // Prepare unions of all the filters - // --------------------------------- - fun unionSubqueriesIfNeeded( - filters: List, - hasher: TagNameValueHasher, - ): RowIdSubQuery? { - val inner = - filters.mapNotNull { filter -> - prepareRowIDSubQueries(filter, hasher) - } - - if (inner.isEmpty()) return null - - return if (inner.size == 1) { - inner.first() - } else { - RowIdSubQuery( - sql = inner.joinToString("\n UNION\n ") { "SELECT row_id FROM (${it.sql})" }, - args = inner.flatMap { it.args }, - ) - } - } - - sealed class TagNameForQuery { - class InTags( - val tagName: String, - ) : TagNameForQuery() - - class AllTags( - val tagName: String, - val tagValueIndex: Int, - ) : TagNameForQuery() - } - - // ---------------------------- - // Inner row id selections - // ---------------------------- - fun prepareRowIDSubQueries( - filter: Filter, - hasher: TagNameValueHasher, - ): RowIdSubQuery? { - if (!filter.isFilledFilter()) return null - - val mustJoinSearch = (filter.search != null) - - val nonDTagsIn = filter.tags?.filter { it.key != "d" } ?: emptyMap() - - val nonDTagsAll = filter.tagsAll?.filter { it.key != "d" } ?: emptyMap() - - val reverseLookup = nonDTagsIn.isNotEmpty() || nonDTagsAll.isNotEmpty() - - val needHeaders = - with(filter) { - (ids != null) || - (authors != null && authors.isNotEmpty()) || - (kinds != null && kinds.isNotEmpty()) || - (tags != null && tags.containsKey("d")) - } - - val hasHeaders = - with(filter) { - (ids != null) || - (authors != null && authors.isNotEmpty()) || - (kinds != null && kinds.isNotEmpty()) || - (tags != null && tags.containsKey("d")) || - (since != null) || - (until != null) || - (limit != null) - } - - var defaultTagKey: TagNameForQuery? = null - - val projection = - buildString { - // always do tags if there are any - if (reverseLookup) { - append("SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags ") - - // it's quite rare to have 2 tags in the filter, but possible - nonDTagsIn.keys.forEachIndexed { index, tagName -> - if (defaultTagKey != null) { - append("INNER JOIN event_tags as event_tagsIn$index ON event_tagsIn$index.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn$index.created_at = event_tags.created_at ") - } else { - defaultTagKey = TagNameForQuery.InTags(tagName) - } - } - - nonDTagsAll.keys.forEachIndexed { index, tagName -> - nonDTagsAll[tagName]!!.forEachIndexed { valueIndex, tagValue -> - if (defaultTagKey != null) { - append("INNER JOIN event_tags as event_tagsAll${index}_$valueIndex ON event_tagsAll${index}_$valueIndex.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll${index}_$valueIndex.created_at = event_tags.created_at ") - } else { - defaultTagKey = TagNameForQuery.AllTags(tagName, valueIndex) - } - } - } - - if (needHeaders) { - append("INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id ") - } - - if (mustJoinSearch) { - append("INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id ") - } - } else if (mustJoinSearch) { - append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName} ") - - if (hasHeaders) { - append("INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") - } - } else { - // no tags and no search. - append("SELECT event_headers.row_id as row_id FROM event_headers ") - } - } - - val clause = - where { - // the order should match indexes - // ids reduce the filter the most - filter.ids?.let { equalsOrIn("event_headers.id", it) } - - // it's quite rare to have 2 tags in the filter, but possible - nonDTagsIn.keys.forEachIndexed { index, tagName -> - val column = - if (defaultTagKey == null || (defaultTagKey is TagNameForQuery.InTags && defaultTagKey.tagName == tagName)) { - "event_tags.tag_hash" - } else { - "event_tagsIn$index.tag_hash" - } - - equalsOrIn( - column, - nonDTagsIn[tagName]!!.map { - hasher.hash(tagName, it) - }, - ) - } - - // there are indexes for these, starting with tags. - nonDTagsAll.keys.forEachIndexed { index, tagName -> - nonDTagsAll[tagName]!!.forEachIndexed { valueIndex, tagValue -> - val column = - if (defaultTagKey == null || (defaultTagKey is TagNameForQuery.AllTags && defaultTagKey.tagName == tagName && defaultTagKey.tagValueIndex == valueIndex)) { - "event_tags.tag_hash" - } else { - "event_tagsAll${index}_$valueIndex.tag_hash" - } - - equals(column, hasher.hash(tagName, tagValue)) - } - } - - // range search is bad but most of the time these are up the top with few elements. - if (reverseLookup) { - filter.since?.let { greaterThanOrEquals("event_tags.created_at", it) } - filter.until?.let { lessThanOrEquals("event_tags.created_at", it) } - } else { - filter.since?.let { greaterThanOrEquals("event_headers.created_at", it) } - filter.until?.let { lessThanOrEquals("event_headers.created_at", it) } - } - - filter.kinds?.let { equalsOrIn("event_headers.kind", it) } - filter.authors?.let { equalsOrIn("event_headers.pubkey", it) } - - // there are indexes for these, starting with tags. - filter.tags?.forEach { (tagName, tagValues) -> - if (tagName == "d") { - equalsOrIn("event_headers.d_tag", tagValues) - } - } - - // if search is included, SQLLite will always start here. - filter.search?.let { - if (it.isNotBlank()) { - match(fts.tableName, it) - } - } - } - - val whereClause = - if (filter.limit != null) { - if (reverseLookup) { - "${clause.conditions} ORDER BY event_tags.created_at DESC LIMIT ${filter.limit}" - } else { - "${clause.conditions} ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT ${filter.limit}" - } - } else { - clause.conditions - } - - return RowIdSubQuery("$projection WHERE $whereClause", clause.args) - } - override fun deleteAll(db: SQLiteDatabase) { db.execSQL("DELETE FROM event_tags") db.execSQL("DELETE FROM event_headers") } - - data class RowIdSubQuery( - val sql: String, - val args: List, - ) } diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt new file mode 100644 index 000000000..0d3a5bb79 --- /dev/null +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryBuilder.kt @@ -0,0 +1,467 @@ +/** + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import android.database.Cursor +import android.database.sqlite.SQLiteDatabase +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.Kind +import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.sqlite.sql.where +import com.vitorpamplona.quartz.utils.EventFactory +import kotlin.collections.component1 +import kotlin.collections.component2 + +class QueryBuilder( + val fts: FullTextSearchModule, + val hasher: (db: SQLiteDatabase) -> TagNameValueHasher, +) { + fun planQuery( + filter: Filter, + hasher: TagNameValueHasher, + db: SQLiteDatabase, + ): String { + val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher) + + return if (rowIdSubQuery == null) { + val query = makeEverythingQuery() + db.explainQuery(query) + } else { + val query = makeQueryIn(rowIdSubQuery.sql) + db.explainQuery(query, rowIdSubQuery.args.toTypedArray()) + } + } + + fun query( + filter: Filter, + db: SQLiteDatabase, + ): List { + val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher(db)) + + return if (rowIdSubQuery == null) { + db.runQuery(makeEverythingQuery()) + } else { + db.runQuery(makeQueryIn(rowIdSubQuery.sql), rowIdSubQuery.args) + } + } + + fun query( + filter: Filter, + db: SQLiteDatabase, + onEach: (T) -> Unit, + ) { + val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher(db)) + + return if (rowIdSubQuery == null) { + db.runQuery(makeEverythingQuery(), onEach = onEach) + } else { + db.runQuery(makeQueryIn(rowIdSubQuery.sql), rowIdSubQuery.args, onEach) + } + } + + fun planQuery( + filters: List, + hasher: TagNameValueHasher, + db: SQLiteDatabase, + ): String { + val rowIdSubQuery = unionSubqueriesIfNeeded(filters, hasher) + + return if (rowIdSubQuery == null) { + val query = makeEverythingQuery() + db.explainQuery(query) + } else { + val query = makeQueryIn(rowIdSubQuery.sql) + db.explainQuery(query, rowIdSubQuery.args.toTypedArray()) + } + } + + fun query( + filters: List, + db: SQLiteDatabase, + ): List { + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return db.runQuery(makeEverythingQuery()) + return db.runQuery(makeQueryIn(rowIdSubqueries.sql), rowIdSubqueries.args) + } + + fun query( + filters: List, + db: SQLiteDatabase, + onEach: (T) -> Unit, + ) { + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) + + if (rowIdSubqueries == null) { + db.runQuery(makeEverythingQuery(), onEach = onEach) + } else { + db.runQuery(makeQueryIn(rowIdSubqueries.sql), rowIdSubqueries.args, onEach) + } + } + + private fun makeEverythingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC, id" + + private fun makeQueryIn(rowIdQuery: String) = + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + $rowIdQuery + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + """.trimIndent() + + private fun SQLiteDatabase.runQuery( + sql: String, + args: List = emptyList(), + ): List = + rawQuery(sql, args.toTypedArray()).use { cursor -> + ArrayList(cursor.count).apply { + while (cursor.moveToNext()) { + add(cursor.toEvent()) + } + } + } + + private inline fun SQLiteDatabase.runQuery( + sql: String, + args: List = emptyList(), + onEach: (T) -> Unit, + ) = rawQuery(sql, args.toTypedArray()).use { cursor -> + while (cursor.moveToNext()) { + onEach(cursor.toEvent()) + } + } + + private fun Cursor.toEvent() = + EventFactory.create( + getString(0).intern(), + getString(1).intern(), + getLong(2), + getInt(3), + OptimizedJsonMapper.fromJsonToTagArray(getString(4)), + getString(5), + getString(6), + ) + + class RawEvent( + val id: HexKey, + val pubKey: HexKey, + val createdAt: Long, + val kind: Kind, + val jsonTags: String, + val content: String, + val sig: HexKey, + ) { + fun toEvent() = + EventFactory.create( + id.intern(), + pubKey.intern(), + createdAt, + kind, + OptimizedJsonMapper.fromJsonToTagArray(jsonTags), + content, + sig, + ) + } + + private fun Cursor.toRawEvent() = + RawEvent( + getString(0), + getString(1), + getLong(2), + getInt(3), + getString(4), + getString(5), + getString(6), + ) + + // -------------- + // Counts + // ------------- + fun count( + filter: Filter, + db: SQLiteDatabase, + ): Int { + val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher(db)) + + return if (rowIdSubQuery == null) { + db.countEverything() + } else { + db.countIn(rowIdSubQuery.sql, rowIdSubQuery.args) + } + } + + fun count( + filters: List, + db: SQLiteDatabase, + ): Int { + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return db.countEverything() + + return db.countIn(rowIdSubqueries.sql, rowIdSubqueries.args) + } + + private fun SQLiteDatabase.countEverything() = runCount("SELECT count(*) as count FROM event_headers") + + private fun SQLiteDatabase.countIn( + rowIdQuery: String, + args: List, + ) = runCount("SELECT COUNT(*) as count FROM ($rowIdQuery)", args) + + private fun SQLiteDatabase.runCount( + sql: String, + args: List = emptyList(), + ): Int = + rawQuery(sql, args.toTypedArray()).use { cursor -> + cursor.moveToNext() + cursor.getInt(0) + } + + // -------------- + // Deletes + // ------------- + fun delete( + filter: Filter, + db: SQLiteDatabase, + ): Int { + val rowIdQuery = prepareRowIDSubQueries(filter, hasher(db)) + + return if (rowIdQuery == null) { + 0 + } else { + db.runDelete(rowIdQuery.sql, rowIdQuery.args) + } + } + + fun delete( + filters: List, + db: SQLiteDatabase, + ): Int { + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return 0 + + return db.runDelete(rowIdSubqueries.sql, rowIdSubqueries.args) + } + + private fun SQLiteDatabase.runDelete( + sql: String, + args: List = emptyList(), + ): Int = delete("event_headers", "row_id IN ($sql)", args.toTypedArray()) + + // --------------------------------- + // Prepare unions of all the filters + // --------------------------------- + fun unionSubqueriesIfNeeded( + filters: List, + hasher: TagNameValueHasher, + ): RowIdSubQuery? { + val inner = + filters.mapNotNull { filter -> + prepareRowIDSubQueries(filter, hasher) + } + + if (inner.isEmpty()) return null + + return if (inner.size == 1) { + inner.first() + } else { + RowIdSubQuery( + sql = inner.joinToString("\n UNION\n ") { "SELECT row_id FROM (${it.sql})" }, + args = inner.flatMap { it.args }, + ) + } + } + + sealed class TagNameForQuery { + class InTags( + val tagName: String, + ) : TagNameForQuery() + + class AllTags( + val tagName: String, + val tagValueIndex: Int, + ) : TagNameForQuery() + } + + // ---------------------------- + // Inner row id selections + // ---------------------------- + fun prepareRowIDSubQueries( + filter: Filter, + hasher: TagNameValueHasher, + ): RowIdSubQuery? { + if (!filter.isFilledFilter()) return null + + val mustJoinSearch = (filter.search != null) + + val nonDTagsIn = filter.tags?.filter { it.key != "d" } ?: emptyMap() + + val nonDTagsAll = filter.tagsAll?.filter { it.key != "d" } ?: emptyMap() + + val reverseLookup = nonDTagsIn.isNotEmpty() || nonDTagsAll.isNotEmpty() + + val needHeaders = + with(filter) { + (ids != null) || + (authors != null && authors.isNotEmpty()) || + (kinds != null && kinds.isNotEmpty()) || + (tags != null && tags.containsKey("d")) + } + + val hasHeaders = + with(filter) { + (ids != null) || + (authors != null && authors.isNotEmpty()) || + (kinds != null && kinds.isNotEmpty()) || + (tags != null && tags.containsKey("d")) || + (since != null) || + (until != null) || + (limit != null) + } + + var defaultTagKey: TagNameForQuery? = null + + val projection = + buildString { + // always do tags if there are any + if (reverseLookup) { + append("SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags ") + + // it's quite rare to have 2 tags in the filter, but possible + nonDTagsIn.keys.forEachIndexed { index, tagName -> + if (defaultTagKey != null) { + append("INNER JOIN event_tags as event_tagsIn$index ON event_tagsIn$index.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn$index.created_at = event_tags.created_at ") + } else { + defaultTagKey = TagNameForQuery.InTags(tagName) + } + } + + nonDTagsAll.keys.forEachIndexed { index, tagName -> + nonDTagsAll[tagName]!!.forEachIndexed { valueIndex, tagValue -> + if (defaultTagKey != null) { + append("INNER JOIN event_tags as event_tagsAll${index}_$valueIndex ON event_tagsAll${index}_$valueIndex.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll${index}_$valueIndex.created_at = event_tags.created_at ") + } else { + defaultTagKey = TagNameForQuery.AllTags(tagName, valueIndex) + } + } + } + + if (needHeaders) { + append("INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id ") + } + + if (mustJoinSearch) { + append("INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id ") + } + } else if (mustJoinSearch) { + append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName} ") + + if (hasHeaders) { + append("INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") + } + } else { + // no tags and no search. + append("SELECT event_headers.row_id as row_id FROM event_headers ") + } + } + + val clause = + where { + // the order should match indexes + // ids reduce the filter the most + filter.ids?.let { equalsOrIn("event_headers.id", it) } + + // it's quite rare to have 2 tags in the filter, but possible + nonDTagsIn.keys.forEachIndexed { index, tagName -> + val column = + if (defaultTagKey == null || (defaultTagKey is TagNameForQuery.InTags && defaultTagKey.tagName == tagName)) { + "event_tags.tag_hash" + } else { + "event_tagsIn$index.tag_hash" + } + + equalsOrIn( + column, + nonDTagsIn[tagName]!!.map { + hasher.hash(tagName, it) + }, + ) + } + + // there are indexes for these, starting with tags. + nonDTagsAll.keys.forEachIndexed { index, tagName -> + nonDTagsAll[tagName]!!.forEachIndexed { valueIndex, tagValue -> + val column = + if (defaultTagKey == null || (defaultTagKey is TagNameForQuery.AllTags && defaultTagKey.tagName == tagName && defaultTagKey.tagValueIndex == valueIndex)) { + "event_tags.tag_hash" + } else { + "event_tagsAll${index}_$valueIndex.tag_hash" + } + + equals(column, hasher.hash(tagName, tagValue)) + } + } + + // range search is bad but most of the time these are up the top with few elements. + if (reverseLookup) { + filter.since?.let { greaterThanOrEquals("event_tags.created_at", it) } + filter.until?.let { lessThanOrEquals("event_tags.created_at", it) } + } else { + filter.since?.let { greaterThanOrEquals("event_headers.created_at", it) } + filter.until?.let { lessThanOrEquals("event_headers.created_at", it) } + } + + filter.kinds?.let { equalsOrIn("event_headers.kind", it) } + filter.authors?.let { equalsOrIn("event_headers.pubkey", it) } + + // there are indexes for these, starting with tags. + filter.tags?.forEach { (tagName, tagValues) -> + if (tagName == "d") { + equalsOrIn("event_headers.d_tag", tagValues) + } + } + + // if search is included, SQLLite will always start here. + filter.search?.let { + if (it.isNotBlank()) { + match(fts.tableName, it) + } + } + } + + val whereClause = + if (filter.limit != null) { + if (reverseLookup) { + "${clause.conditions} ORDER BY event_tags.created_at DESC LIMIT ${filter.limit}" + } else { + "${clause.conditions} ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT ${filter.limit}" + } + } else { + clause.conditions + } + + return RowIdSubQuery("$projection WHERE $whereClause", clause.args) + } + + data class RowIdSubQuery( + val sql: String, + val args: List, + ) +} diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index c1e72a3bb..c51d50753 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -57,6 +57,8 @@ class SQLiteEventStore( val expirationModule = ExpirationModule() val rightToVanishModule = RightToVanishModule(seedModule::hasher) + val queryBuilder = QueryBuilder(fullTextSearchModule, seedModule::hasher) + val modules = listOf( seedModule, @@ -171,30 +173,30 @@ class SQLiteEventStore( } } - fun query(filter: Filter): List = eventIndexModule.query(filter, readableDatabase) + fun query(filter: Filter): List = queryBuilder.query(filter, readableDatabase) - fun query(filters: List): List = eventIndexModule.query(filters, readableDatabase) + fun query(filters: List): List = queryBuilder.query(filters, readableDatabase) fun query( filter: Filter, onEach: (T) -> Unit, - ) = eventIndexModule.query(filter, readableDatabase, onEach) + ) = queryBuilder.query(filter, readableDatabase, onEach) fun query( filters: List, onEach: (T) -> Unit, - ) = eventIndexModule.query(filters, readableDatabase, onEach) + ) = queryBuilder.query(filters, readableDatabase, onEach) - fun count(filter: Filter): Int = eventIndexModule.count(filter, readableDatabase) + fun count(filter: Filter): Int = queryBuilder.count(filter, readableDatabase) - fun count(filters: List): Int = eventIndexModule.count(filters, readableDatabase) + fun count(filters: List): Int = queryBuilder.count(filters, readableDatabase) fun delete(filter: Filter) { - eventIndexModule.delete(filter, writableDatabase) + queryBuilder.delete(filter, writableDatabase) } fun delete(filters: List) { - eventIndexModule.delete(filters, writableDatabase) + queryBuilder.delete(filters, writableDatabase) } fun delete(id: HexKey): Int = writableDatabase.delete("event_headers", "id = ?", arrayOf(id))