diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableTest.kt index 540833e51..6214aecc6 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableTest.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent import com.vitorpamplona.quartz.utils.TimeUtils import junit.framework.TestCase +import junit.framework.TestCase.assertEquals import junit.framework.TestCase.fail import org.junit.After import org.junit.Before @@ -41,15 +42,12 @@ class AddressableTest { @Before fun setup() { val context = ApplicationProvider.getApplicationContext() - context.deleteDatabase("test.db") - db = EventStore(context, "test.db", relayUrl = "testUrl") + db = EventStore(context, null) } @After fun tearDown() { db.close() - val context = ApplicationProvider.getApplicationContext() - context.deleteDatabase("test.db") } @Test @@ -59,20 +57,25 @@ class AddressableTest { val version2 = signer.sign(LongTextNoteEvent.build("my cool blog, version 2", "title", dTag = "my-cool-blog", createdAt = time + 1)) val version3 = signer.sign(LongTextNoteEvent.build("my cool blog, version 3", "title", dTag = "my-cool-blog", createdAt = time + 2)) + val addressableQuery = Filter(kinds = listOf(version1.kind), authors = listOf(version1.pubKey), tags = mapOf("d" to listOf(version1.dTag()))) + db.insert(version1) db.assertQuery(version1, Filter(ids = listOf(version1.id))) + db.assertQuery(version1, addressableQuery) db.insert(version2) db.assertQuery(null, Filter(ids = listOf(version1.id))) db.assertQuery(version2, Filter(ids = listOf(version2.id))) + db.assertQuery(version2, addressableQuery) db.insert(version3) db.assertQuery(null, Filter(ids = listOf(version1.id))) db.assertQuery(null, Filter(ids = listOf(version2.id))) db.assertQuery(version3, Filter(ids = listOf(version3.id))) + db.assertQuery(version3, addressableQuery) } @Test @@ -82,6 +85,8 @@ class AddressableTest { val version2 = signer.sign(LongTextNoteEvent.build("my cool blog, version 2", "title", dTag = "my-cool-blog", createdAt = time + 1)) val version3 = signer.sign(LongTextNoteEvent.build("my cool blog, version 3", "title", dTag = "my-cool-blog", createdAt = time + 2)) + val addressableQuery = Filter(kinds = listOf(version1.kind), authors = listOf(version1.pubKey), tags = mapOf("d" to listOf(version1.dTag()))) + db.insert(version3) db.assertQuery(version3, Filter(ids = listOf(version3.id))) @@ -101,7 +106,38 @@ class AddressableTest { } db.assertQuery(version3, Filter(ids = listOf(version3.id))) + db.assertQuery(version3, addressableQuery) db.assertQuery(null, Filter(ids = listOf(version2.id))) db.assertQuery(null, Filter(ids = listOf(version1.id))) } + + @Test + fun testTriggersIndexUsage() { + val explainer = + db.store.explainQuery( + """ + SELECT * FROM event_headers + WHERE + event_headers.kind = 30000 AND + event_headers.pubkey = 'aa' AND + event_headers.d_tag = 'test-tag' AND + event_headers.created_at < 1766686500 AND + event_headers.kind >= 30000 AND event_headers.kind < 40000 + """.trimIndent(), + ) + + assertEquals( + """ + SELECT * FROM event_headers + WHERE + event_headers.kind = 30000 AND + event_headers.pubkey = 'aa' AND + event_headers.d_tag = 'test-tag' AND + event_headers.created_at < 1766686500 AND + event_headers.kind >= 30000 AND event_headers.kind < 40000 + └── SEARCH event_headers USING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?) + """.trimIndent(), + explainer, + ) + } } diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt index 02789a115..1f7f771b3 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt @@ -23,12 +23,15 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import android.content.Context import android.database.sqlite.SQLiteConstraintException import androidx.test.core.app.ApplicationProvider +import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent +import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.utils.TimeUtils +import junit.framework.TestCase import junit.framework.TestCase.fail import org.junit.After import org.junit.Assert.assertEquals @@ -170,4 +173,240 @@ class DeletionTest { db.assertQuery(null, Filter(ids = listOf(note2.id))) db.assertQuery(null, Filter(ids = listOf(note3.id))) } + + @Test + fun testInsertDeleteWrap() { + val me = NostrSignerSync() + val myFriend = NostrSignerSync() + + val note1 = me.sign(TextNoteEvent.build("test1")) + val wrap1 = GiftWrapEvent.create(note1, me.pubKey) + val wrap2 = GiftWrapEvent.create(note1, myFriend.pubKey) + + db.insert(wrap1) + db.insert(wrap2) + + db.assertQuery(wrap1, Filter(ids = listOf(wrap1.id))) + db.assertQuery(wrap2, Filter(ids = listOf(wrap2.id))) + + val randomDeletionToWrap = signer.sign(DeletionEvent.build(listOf(wrap1))) + + db.insert(randomDeletionToWrap) + + db.assertQuery(randomDeletionToWrap, Filter(ids = listOf(randomDeletionToWrap.id))) + db.assertQuery(wrap1, Filter(ids = listOf(wrap1.id))) + db.assertQuery(wrap2, Filter(ids = listOf(wrap2.id))) + + val deletion = me.sign(DeletionEvent.build(listOf(wrap1))) + + db.insert(deletion) + + db.assertQuery(deletion, Filter(ids = listOf(deletion.id))) + db.assertQuery(null, Filter(ids = listOf(wrap1.id))) + db.assertQuery(wrap2, Filter(ids = listOf(wrap2.id))) + + // trying to insert again should fail. + try { + db.insert(wrap1) + fail("Should not be able to insert a deleted event") + } catch (e: SQLiteConstraintException) { + assertEquals("blocked: a deletion event exists (code 1811 SQLITE_CONSTRAINT_TRIGGER)", e.message) + } + + db.assertQuery(deletion, Filter(ids = listOf(deletion.id))) + db.assertQuery(null, Filter(ids = listOf(wrap1.id))) + db.assertQuery(wrap2, Filter(ids = listOf(wrap2.id))) + } + + @Test + fun testTriggersIndexUsage() { + val sql = + """ + SELECT 1 FROM event_tags + INNER JOIN event_headers + ON event_headers.row_id = event_tags.event_header_row_id + WHERE + event_tags.tag_hash IN (3221122, 223322) AND + event_headers.kind = 5 AND + event_headers.created_at >= 1766686500 AND + event_headers.pubkey_owner_hash = 22332323 + """.trimIndent() + + val explainer = + db.store.explainQuery(sql) + + TestCase.assertEquals( + """ + ${sql.replace("\n","\n ")} + ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + └── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + """.trimIndent(), + explainer, + ) + } + + @Test + fun testDeleteById() { + val sql = + db.store.deletionModule + .deleteSQL( + pubkey = "key1", + idValues = listOf("ca29c211f", "ca29c211d"), + addresses = emptyList(), + hasher = TagNameValueHasher(0), + ).first() + + TestCase.assertEquals( + """ + DELETE FROM event_headers + WHERE + id IN ("ca29c211f","ca29c211d") AND + pubkey_owner_hash = "1573573083296714675" + ├── SEARCH event_headers USING INDEX event_headers_id (id=?) + ├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?) + ├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?) + └── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?) + """.trimIndent(), + db.store.explainQuery(sql.sql, sql.args), + ) + } + + @Test + fun testDeleteAddressable() { + val sql = + db.store.deletionModule + .deleteSQL( + pubkey = "key1", + idValues = emptyList(), + addresses = + listOf( + Address(30000, "key1", "a"), + ), + hasher = TagNameValueHasher(0), + ).first() + + TestCase.assertEquals( + """ + DELETE FROM event_headers + WHERE ( + (kind = "30000" AND pubkey = "key1" AND d_tag = "a") + ) AND + kind >= 30000 AND kind < 40000 + ├── SEARCH event_headers USING COVERING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?) + ├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?) + ├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?) + └── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?) + """.trimIndent(), + db.store.explainQuery(sql.sql, sql.args), + ) + } + + @Test + fun testDeleteAddressablesSingleKind() { + val sql = + db.store.deletionModule + .deleteSQL( + pubkey = "key1", + idValues = emptyList(), + addresses = + listOf( + Address(30000, "key1", "a"), + Address(30000, "key1", "b"), + Address(30000, "key1", "c"), + Address(30000, "key1", "d"), + ), + hasher = TagNameValueHasher(0), + ).first() + + TestCase.assertEquals( + """ + DELETE FROM event_headers + WHERE ( + (kind = "30000" AND pubkey = "key1" AND d_tag IN ("a","b","c","d")) + ) AND + kind >= 30000 AND kind < 40000 + ├── SEARCH event_headers USING COVERING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?) + ├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?) + ├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?) + └── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?) + """.trimIndent(), + db.store.explainQuery(sql.sql, sql.args), + ) + } + + @Test + fun testDeleteAddressablesMultipleKinds() { + val sql = + db.store.deletionModule + .deleteSQL( + pubkey = "key1", + idValues = emptyList(), + addresses = + listOf( + Address(30000, "key1", "a"), + Address(30000, "key1", "b"), + Address(30101, "key1", "c"), + Address(30101, "key1", "d"), + Address(30001, "key2", "e"), + Address(30001, "key2", "f"), + ), + hasher = TagNameValueHasher(0), + ).first() + + TestCase.assertEquals( + """ + DELETE FROM event_headers + WHERE ( + (kind = "30000" AND pubkey = "key1" AND d_tag IN ("a","b")) + OR + (kind = "30101" AND pubkey = "key1" AND d_tag IN ("c","d")) + ) AND + kind >= 30000 AND kind < 40000 + ├── MULTI-INDEX OR + │ ├── INDEX 1 + │ │ └── SEARCH event_headers USING COVERING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?) + │ └── INDEX 2 + │ └── SEARCH event_headers USING COVERING INDEX addressable_idx (kind=? AND pubkey=? AND d_tag=?) + ├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?) + ├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?) + └── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?) + """.trimIndent(), + db.store.explainQuery(sql.sql, sql.args), + ) + } + + @Test + fun testDeleteReplaceables() { + val sql = + db.store.deletionModule + .deleteSQL( + pubkey = "key1", + idValues = emptyList(), + addresses = + listOf( + Address(10000, "key1", ""), + Address(10000, "key1", ""), + Address(10001, "key1", ""), + Address(10001, "key1", ""), + Address(10001, "key2", ""), + Address(10001, "key2", ""), + ), + hasher = TagNameValueHasher(0), + ).first() + + TestCase.assertEquals( + """ + DELETE FROM event_headers + WHERE + kind IN ("10000","10001") AND + pubkey = "key1" AND + ((kind in (0,3)) OR (kind >= 10000 AND kind < 20000)) + ├── SEARCH event_headers USING COVERING INDEX replaceable_idx (kind=? AND pubkey=?) + ├── SEARCH event_vanish USING INTEGER PRIMARY KEY (rowid=?) + ├── SEARCH event_expirations USING INTEGER PRIMARY KEY (rowid=?) + └── SEARCH event_tags USING COVERING INDEX fk_event_tags_header_id (event_header_row_id=?) + """.trimIndent(), + db.store.explainQuery(sql.sql, sql.args), + ) + } } diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationTest.kt index add6ce20b..6197a8967 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationTest.kt @@ -42,7 +42,7 @@ class ExpirationTest { @Before fun setup() { val context = ApplicationProvider.getApplicationContext() - db = EventStore(context, null, relayUrl = "testUrl") + db = EventStore(context, null) } @After diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/LargeDBTests.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/LargeDBTests.kt index 53277c750..a16e2909b 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/LargeDBTests.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/LargeDBTests.kt @@ -56,15 +56,12 @@ class LargeDBTests { @Before fun setup() { val context = ApplicationProvider.getApplicationContext() - context.deleteDatabase("test_large.db") - db = EventStore(context, "largeDBTest.db") + db = EventStore(context, null) } @After fun tearDown() { db.close() - val context = ApplicationProvider.getApplicationContext() - context.deleteDatabase("test_large.db") } @Test diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt new file mode 100644 index 000000000..2a3983a8d --- /dev/null +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt @@ -0,0 +1,480 @@ +/** + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip02FollowList.ContactListEvent +import junit.framework.TestCase +import org.junit.After +import org.junit.Assert +import org.junit.Before +import org.junit.Test + +class QueryAssemblerTest { + val hasher = TagNameValueHasher(0L) + val builder = EventIndexesModule(FullTextSearchModule(), { hasher }) + + val key1 = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" + val key2 = "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14" + val key3 = "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9" + + private lateinit var db: EventStore + + @Before + fun setup() { + val context = ApplicationProvider.getApplicationContext() + db = EventStore(context, null) + } + + @After + fun tearDown() { + db.close() + } + + fun explain(f: Filter) = builder.planQuery(f, hasher, db.store.readableDatabase) + + fun explain(f: List) = builder.planQuery(f, hasher, db.store.readableDatabase) + + @Test + fun testEmpty() { + Assert.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC, id + └── SCAN event_headers USING INDEX query_by_created_at_id + """.trimIndent(), + explain(Filter()), + ) + } + + @Test + fun testCheckDeletionEventExists() { + val query = + explain( + Filter( + kinds = listOf(5), + authors = listOf(key1), + tags = mapOf("e" to listOf(key2)), + since = 1750889190, + ), + ) + + println(query) + + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_tags.event_header_row_id as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_headers.created_at >= "1750889190") AND (event_tags.tag_hash = "2657743813502222172") AND (event_headers.kind = "5") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + query, + ) + } + + @Test + fun testLimit() { + val explainer = explain(Filter(limit = 10)) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_headers.row_id as row_id FROM event_headers WHERE 1 = 1 ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 10 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + explainer, + ) + } + + @Test + fun testLimits() { + val explainer = explain(listOf(Filter(limit = 10), Filter(limit = 30))) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers WHERE 1 = 1 ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 10) + UNION + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers WHERE 1 = 1 ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30) + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ └── COMPOUND QUERY + │ ├── LEFT-MOST SUBQUERY + │ │ ├── CO-ROUTINE (subquery-1) + │ │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id + │ │ └── SCAN (subquery-1) + │ └── UNION USING TEMP B-TREE + │ ├── CO-ROUTINE (subquery-3) + │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id + │ └── SCAN (subquery-3) + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + explainer, + ) + } + + @Test + fun testAllFeatures() { + val sql = + explain( + listOf( + Filter(limit = 10), + Filter( + authors = listOf(key1), + kinds = listOf(1, 1111), + search = "keywords", + limit = 100, + ), + Filter(kinds = listOf(20), search = "cats", limit = 30), + ), + ) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT row_id FROM (SELECT event_headers.row_id as row_id FROM event_headers WHERE 1 = 1 ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 10) + UNION + SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111")) AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") AND (event_fts MATCH "keywords") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 100) + UNION + SELECT row_id FROM (SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind = "20") AND (event_fts MATCH "cats") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30) + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ └── COMPOUND QUERY + │ ├── LEFT-MOST SUBQUERY + │ │ ├── CO-ROUTINE (subquery-1) + │ │ │ └── SCAN event_headers USING COVERING INDEX query_by_created_at_id + │ │ └── SCAN (subquery-1) + │ ├── UNION USING TEMP B-TREE + │ │ ├── CO-ROUTINE (subquery-3) + │ │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + │ │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ │ │ └── USE TEMP B-TREE FOR ORDER BY + │ │ └── SCAN (subquery-3) + │ └── UNION USING TEMP B-TREE + │ ├── CO-ROUTINE (subquery-5) + │ │ ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + │ │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ │ └── USE TEMP B-TREE FOR ORDER BY + │ └── SCAN (subquery-5) + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testKind() { + val sql = + explain( + listOf( + Filter( + kinds = listOf(ContactListEvent.KIND), + limit = 30, + ), + ), + ) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_headers.row_id as row_id FROM event_headers WHERE event_headers.kind = "3" ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_dtag_idx (kind=?) + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testKindAndDTag() { + val sql = + explain( + listOf( + Filter( + kinds = listOf(ContactListEvent.KIND), + tags = mapOf("d" to listOf("")), + limit = 30, + ), + ), + ) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_headers.row_id as row_id FROM event_headers WHERE (event_headers.kind = "3") AND (event_headers.d_tag = "") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_dtag_idx (kind=?) + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testFollowersOf() { + val sql = + explain( + listOf( + Filter( + tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), + limit = 30, + ), + ), + ) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_tags.event_header_row_id as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE event_tags.tag_hash = "-4551135004136952885" ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testTagsAndKinds() { + val sql = + explain( + listOf( + Filter( + kinds = listOf(ContactListEvent.KIND), + tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), + limit = 30, + ), + ), + ) + + println(sql) + + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_tags.event_header_row_id as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testTagsAndAuthors() { + val sql = + explain( + listOf( + Filter( + authors = listOf(key1), + tags = mapOf("p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c")), + limit = 30, + ), + ), + ) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_tags.event_header_row_id as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testTwoTags() { + val sql = + explain( + listOf( + Filter( + kinds = listOf(1), + tags = + mapOf( + "p" to listOf("460c25e682fda7832b52d1f22d3d22b3176d972f60dcdc3212ed8c92ef85065c"), + "t" to listOf("hashtag"), + ), + limit = 30, + ), + ), + ) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_tags.event_header_row_id as row_id FROM event_tags INNER JOIN event_tags as event_tagst ON event_tagst.event_header_row_id = event_tags.event_header_row_id INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tagst.tag_hash = "-6379614208644810021") AND (event_headers.kind = "1") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?) + │ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + │ ├── SEARCH event_tagst USING COVERING INDEX query_by_tags_hash (tag_hash=? AND event_header_row_id=?) + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testIdQuery() { + val sql = explain(Filter(ids = listOf(key1))) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_headers.row_id as row_id FROM event_headers WHERE event_headers.id = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── SEARCH event_headers USING COVERING INDEX event_headers_id (id=?) + └── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + """.trimIndent(), + sql, + ) + } + + @Test + fun testAuthors() { + val sql = explain(Filter(authors = listOf(key1, key2), kinds = listOf(1, 30023), limit = 300)) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_headers.row_id as row_id FROM event_headers WHERE (event_headers.kind IN ("1", "30023")) AND (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14")) ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 300 + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── CO-ROUTINE filtered + │ ├── SEARCH event_headers USING INDEX query_by_kind_pubkey_dtag_idx (kind=? AND pubkey=?) + │ └── USE TEMP B-TREE FOR ORDER BY + ├── SCAN filtered + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testAuthorsAndSearch() { + val sql = explain(Filter(authors = listOf(key1, key2, key3), search = "keywords")) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.pubkey IN ("7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d", "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14", "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9")) AND (event_fts MATCH "keywords") + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } + + @Test + fun testKindAndSearch() { + val sql = explain(Filter(kinds = listOf(1, 1111, 10000), search = "keywords")) + TestCase.assertEquals( + """ + SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers + INNER JOIN ( + SELECT event_fts.event_header_row_id as row_id FROM event_fts INNER JOIN event_headers ON event_headers.row_id = event_fts.event_header_row_id WHERE (event_headers.kind IN ("1", "1111", "10000")) AND (event_fts MATCH "keywords") + ) AS filtered + ON event_headers.row_id = filtered.row_id + ORDER BY created_at DESC, id + ├── SCAN event_fts VIRTUAL TABLE INDEX 4: + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?) + └── USE TEMP B-TREE FOR ORDER BY + """.trimIndent(), + sql, + ) + } +} diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableTest.kt index 0a85cfc75..97c8aa75d 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableTest.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.utils.TimeUtils import junit.framework.TestCase +import junit.framework.TestCase.assertEquals import junit.framework.TestCase.fail import org.junit.After import org.junit.Before @@ -41,7 +42,7 @@ class ReplaceableTest { @Before fun setup() { val context = ApplicationProvider.getApplicationContext() - db = EventStore(context, null, relayUrl = "testUrl") + db = EventStore(context, null) } @After @@ -56,20 +57,25 @@ class ReplaceableTest { val version2 = signer.sign(MetadataEvent.createNew("Vitor 2", createdAt = time + 1)) val version3 = signer.sign(MetadataEvent.createNew("Vitor 3", createdAt = time + 2)) + val addressableQuery = Filter(kinds = listOf(version1.kind), authors = listOf(version1.pubKey), tags = mapOf("d" to listOf(version1.dTag()))) + db.insert(version1) db.assertQuery(version1, Filter(ids = listOf(version1.id))) + db.assertQuery(version1, addressableQuery) db.insert(version2) db.assertQuery(null, Filter(ids = listOf(version1.id))) db.assertQuery(version2, Filter(ids = listOf(version2.id))) + db.assertQuery(version2, addressableQuery) db.insert(version3) db.assertQuery(null, Filter(ids = listOf(version1.id))) db.assertQuery(null, Filter(ids = listOf(version2.id))) db.assertQuery(version3, Filter(ids = listOf(version3.id))) + db.assertQuery(version3, addressableQuery) } @Test @@ -79,6 +85,8 @@ class ReplaceableTest { val version2 = signer.sign(MetadataEvent.createNew("Vitor 2", createdAt = time + 1)) val version3 = signer.sign(MetadataEvent.createNew("Vitor 3", createdAt = time + 2)) + val addressableQuery = Filter(kinds = listOf(version1.kind), authors = listOf(version1.pubKey), tags = mapOf("d" to listOf(version1.dTag()))) + db.insert(version3) db.assertQuery(version3, Filter(ids = listOf(version3.id))) @@ -98,7 +106,64 @@ class ReplaceableTest { } db.assertQuery(version3, Filter(ids = listOf(version3.id))) + db.assertQuery(version3, addressableQuery) db.assertQuery(null, Filter(ids = listOf(version2.id))) db.assertQuery(null, Filter(ids = listOf(version1.id))) } + + @Test + fun testTriggersIndexUsageKind0() { + val explainer = + db.store.explainQuery( + """ + SELECT * FROM event_headers + WHERE + event_headers.kind = 0 AND + event_headers.pubkey = 'aa' AND + event_headers.created_at < 1766686500 AND + ((event_headers.kind IN (0, 3)) OR (event_headers.kind >= 10000 AND event_headers.kind < 20000)); + """.trimIndent(), + ) + + assertEquals( + """ + SELECT * FROM event_headers + WHERE + event_headers.kind = 0 AND + event_headers.pubkey = 'aa' AND + event_headers.created_at < 1766686500 AND + ((event_headers.kind IN (0, 3)) OR (event_headers.kind >= 10000 AND event_headers.kind < 20000)); + └── SEARCH event_headers USING INDEX replaceable_idx (kind=? AND pubkey=?) + """.trimIndent(), + explainer, + ) + } + + @Test + fun testTriggersIndexUsageKind3() { + val explainer = + db.store.explainQuery( + """ + SELECT * FROM event_headers + WHERE + event_headers.kind = 3 AND + event_headers.pubkey = 'aa' AND + event_headers.created_at < 1766686500 AND + ((event_headers.kind IN (0, 3)) OR (event_headers.kind >= 10000 AND event_headers.kind < 20000)); + """.trimIndent(), + ) + + assertEquals( + """ + SELECT * FROM event_headers + WHERE + event_headers.kind = 3 AND + event_headers.pubkey = 'aa' AND + event_headers.created_at < 1766686500 AND + ((event_headers.kind IN (0, 3)) OR (event_headers.kind >= 10000 AND event_headers.kind < 20000)); + └── SEARCH event_headers USING INDEX replaceable_idx (kind=? AND pubkey=?) + """.trimIndent(), + explainer, + ) + } } diff --git a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/RightToVanishTest.kt b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/RightToVanishTest.kt index 578d6aeb7..1a7f7aad5 100644 --- a/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/RightToVanishTest.kt +++ b/quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/RightToVanishTest.kt @@ -26,6 +26,7 @@ import androidx.test.core.app.ApplicationProvider import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.TimeUtils import junit.framework.TestCase.fail @@ -87,4 +88,49 @@ class RightToVanishTest { db.assertQuery(null, Filter(ids = listOf(note2.id))) db.assertQuery(note3, Filter(ids = listOf(note3.id))) } + + @Test + fun testInsertDeleteGiftWrap() { + val time = TimeUtils.now() + + val me = NostrSignerSync() + val myFriend = NostrSignerSync() + + val note1 = me.sign(TextNoteEvent.build("test1", createdAt = time)) + val wrap1 = GiftWrapEvent.create(note1, me.pubKey) + val wrap2 = GiftWrapEvent.create(note1, myFriend.pubKey) + + db.insert(wrap1) + db.insert(wrap2) + + db.assertQuery(wrap1, Filter(ids = listOf(wrap1.id))) + db.assertQuery(wrap2, Filter(ids = listOf(wrap2.id))) + + val randomVanishToWrap = signer.sign(RequestToVanishEvent.build("testUrl", createdAt = time + 2)) + + db.insert(randomVanishToWrap) + + db.assertQuery(wrap1, Filter(ids = listOf(wrap1.id))) + db.assertQuery(wrap2, Filter(ids = listOf(wrap2.id))) + + val vanish = me.sign(RequestToVanishEvent.build("testUrl", createdAt = time + 2)) + + db.insert(vanish) + + db.assertQuery(vanish, Filter(ids = listOf(vanish.id))) + db.assertQuery(null, Filter(ids = listOf(wrap1.id))) + db.assertQuery(wrap2, Filter(ids = listOf(wrap2.id))) + + // trying to insert again should fail. + try { + db.insert(wrap1) + fail("Should not be able to insert a deleted event") + } catch (e: SQLiteConstraintException) { + assertEquals("blocked: a request to vanish event exists (code 1811 SQLITE_CONSTRAINT_TRIGGER)", e.message) + } + + db.assertQuery(vanish, Filter(ids = listOf(vanish.id))) + db.assertQuery(null, Filter(ids = listOf(wrap1.id))) + db.assertQuery(wrap2, Filter(ids = listOf(wrap2.id))) + } } diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableModule.kt index 27f625d66..735997518 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/AddressableModule.kt @@ -36,6 +36,8 @@ class AddressableModule : IModule { // if a newer addressable is inserted the unique index // above will be triggered. Delete cascade will take // care of the event_tags table + // the duplicate: kind >= 30000 AND kind < 40000 + // helps SQLlite find the index above db.execSQL( """ CREATE TRIGGER delete_older_addressable_event @@ -48,7 +50,8 @@ class AddressableModule : IModule { event_headers.kind = NEW.kind AND event_headers.pubkey = NEW.pubkey AND event_headers.d_tag = NEW.d_tag AND - event_headers.created_at < NEW.created_at; + event_headers.created_at < NEW.created_at AND + event_headers.kind >= 30000 AND event_headers.kind < 40000; END; """.trimIndent(), ) diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt index 51c81d6ee..32e567fd1 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt @@ -21,12 +21,22 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import android.database.sqlite.SQLiteDatabase +import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.isAddressable +import com.vitorpamplona.quartz.nip01Core.core.isReplaceable import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent -class DeletionRequestModule : IModule { +class DeletionRequestModule( + val hasher: (db: SQLiteDatabase) -> TagNameValueHasher, +) : IModule { + /** + * Creates a trigger to reject events that have been + * deleted by ID or ATag including GiftWraps that + * must be checked against the p-tag (pubkey_owner_hash) + */ override fun create(db: SQLiteDatabase) { - // rejects deleted events. db.execSQL( """ CREATE TRIGGER reject_deleted_events @@ -36,14 +46,14 @@ class DeletionRequestModule : IModule { -- Check for ID-based deletion record SELECT RAISE(ABORT, 'blocked: a deletion event exists') WHERE EXISTS ( - SELECT 1 FROM event_headers - INNER JOIN event_tags + SELECT 1 FROM event_tags + INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE - event_headers.created_at >= NEW.created_at AND + event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND event_headers.kind = 5 AND - event_headers.pubkey = NEW.pubkey AND - event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) + event_headers.pubkey_owner_hash = NEW.pubkey_owner_hash AND + event_headers.created_at >= NEW.created_at ); END; """.trimIndent(), @@ -56,36 +66,110 @@ class DeletionRequestModule : IModule { fun insert( event: Event, - headerId: Long, db: SQLiteDatabase, ) { if (event is DeletionEvent) { val idValues = event.deleteEventIds() - val idParams = idValues.joinToString(",") { "?" } - val addresses = event.deleteAddresses() - val addressParams = addresses.joinToString(",") { "(?, ?)" } - val addressValues = addresses.flatMap { listOf(it.kind, it.dTag) } - val whereClause = - if (idValues.isNotEmpty() && addresses.isNotEmpty()) { - "(id IN ($idParams) OR (kind, d_tag) IN ($addressParams)) AND pubkey = ?" - } else if (idValues.isNotEmpty()) { - "id IN ($idParams) AND pubkey = ?" - } else if (addresses.isNotEmpty()) { - "(kind, d_tag) IN ($addressParams) AND pubkey = ?" - } else { - return - } - val whereParams = idValues.plus(addressValues).plus(event.pubKey).toTypedArray() - - db.execSQL( - """ - DELETE FROM event_headers - WHERE $whereClause; - """.trimIndent(), - whereParams, - ) + deleteSQL(event.pubKey, idValues, addresses, hasher(db)).forEach { delete -> + db.execSQL(delete.sql, delete.args) + } } } + + /** + * Creates a Delete statement that correctly deletes by id, + * by address and by replaceable (no d-tag) using each index + * appropriately, including GiftWraps where the owner is the + * p-tag (via event_header.pubkey_owner_hash) + */ + fun deleteSQL( + pubkey: HexKey, + idValues: List, + addresses: List
, + hasher: TagNameValueHasher, + ): List { + val owner = hasher.hash(pubkey) + val idParams = idValues.joinToString(",") { "?" } + + // aligns each type of param with the need to filter d-tag + // and thus each index type + val addressablesByKind = addresses.filter { it.kind.isAddressable() && it.pubKeyHex == pubkey }.groupBy { it.kind } + val replaceablesByKind = addresses.filter { it.kind.isReplaceable() && it.pubKeyHex == pubkey }.groupBy { it.kind } + + val addressableParams = + addressablesByKind.keys.joinToString("\n OR\n ") { + val tagList = addressablesByKind[it] + if (tagList == null) { + "" + } else if (tagList.size == 1) { + "(kind = ? AND pubkey = ? AND d_tag = ?)" + } else { + "(kind = ? AND pubkey = ? AND d_tag IN (${tagList.joinToString(",") { "?" }}))" + } + } + + val addressableValues = + addressablesByKind.flatMap { + listOf(it.key.toLong(), pubkey) + it.value.map { it.dTag } + } + + val replaceableKindsParam = replaceablesByKind.keys.joinToString(",") { "?" } + val replaceableKindsValues = replaceablesByKind.keys.map { it.toLong() } + + val deleteById = + if (idValues.isNotEmpty()) { + SqlArgs( + """ + DELETE FROM event_headers + WHERE + id IN ($idParams) AND + pubkey_owner_hash = ? + """.trimIndent(), + idValues.plus(owner).toTypedArray(), + ) + } else { + null + } + + val deleteByAddress = + if (addressableValues.isNotEmpty()) { + SqlArgs( + """ + DELETE FROM event_headers + WHERE ( + $addressableParams + ) AND + kind >= 30000 AND kind < 40000 + """.trimIndent(), + addressableValues.toTypedArray(), + ) + } else { + null + } + + val deleteByReplaceable = + if (replaceableKindsParam.isNotEmpty()) { + SqlArgs( + """ + DELETE FROM event_headers + WHERE + kind IN ($replaceableKindsParam) AND + pubkey = ? AND + ((kind in (0,3)) OR (kind >= 10000 AND kind < 20000)) + """.trimIndent(), + replaceableKindsValues.plus(pubkey).toTypedArray(), + ) + } else { + null + } + + return listOfNotNull(deleteById, deleteByAddress, deleteByReplaceable) + } + + class SqlArgs( + val sql: String, + val args: Array, + ) } diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt index f8ceb93cc..22f9af3ab 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt @@ -29,17 +29,14 @@ import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.core.Tag import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.store.sqlite.sql.where +import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.utils.EventFactory class EventIndexesModule( val fts: FullTextSearchModule, - val seedModule: SeedModule, + val hasher: (db: SQLiteDatabase) -> TagNameValueHasher, val tagIndexStrategy: IndexingStrategy = IndexingStrategy(), ) : IModule { - private var hasherCache: TagNameValueHasher? = null - - fun hasher(db: SQLiteDatabase): TagNameValueHasher = hasherCache ?: TagNameValueHasher(seedModule.getSeed(db)).also { hasherCache = it } - override fun create(db: SQLiteDatabase) { db.execSQL( """ @@ -50,11 +47,12 @@ class EventIndexesModule( created_at INTEGER NOT NULL, kind INTEGER NOT NULL, d_tag TEXT, - etag_hash INTEGER NOT NULL, - atag_hash INTEGER, tags TEXT NOT NULL, content TEXT NOT NULL, - sig TEXT NOT NULL + sig TEXT NOT NULL, + pubkey_owner_hash INTEGER NOT NULL, + etag_hash INTEGER, + atag_hash INTEGER ) """.trimIndent(), ) @@ -69,10 +67,14 @@ class EventIndexesModule( """.trimIndent(), ) - db.execSQL("CREATE UNIQUE INDEX event_headers_id ON event_headers (id)") - db.execSQL("CREATE INDEX query_by_kind_pubkey_idx ON event_headers (created_at desc, kind, pubkey, d_tag)") - db.execSQL("CREATE INDEX query_by_id_idx ON event_headers (created_at desc, id)") - db.execSQL("CREATE INDEX query_by_tags_idx ON event_tags (tag_hash)") + db.execSQL("CREATE UNIQUE INDEX event_headers_id ON event_headers (id)") + db.execSQL("CREATE INDEX query_by_kind_pubkey_dtag_idx ON event_headers (kind, pubkey, d_tag)") + db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers (created_at desc, id)") + // need to check if this is actually needed. + db.execSQL("CREATE INDEX query_by_created_at_kind_key ON event_headers (created_at desc, kind, pubkey)") + + db.execSQL("CREATE INDEX fk_event_tags_header_id ON event_tags (event_header_row_id)") + db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, event_header_row_id)") // Prevent updates to maintain immutability db.execSQL( @@ -106,9 +108,9 @@ class EventIndexesModule( val sqlInsertHeader = """ INSERT INTO event_headers - (id, pubkey, created_at, kind, tags, content, sig, d_tag, etag_hash, atag_hash) + (id, pubkey, created_at, kind, tags, content, sig, d_tag, pubkey_owner_hash, etag_hash, atag_hash) VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """.trimIndent() val sqlInsertTags = @@ -125,35 +127,57 @@ class EventIndexesModule( ): Long { val hasher = hasher(db) val stmt = db.compileStatement(sqlInsertHeader) + + val kindLong = event.kind.toLong() + val pubkeyHash = hasher.hash(event.pubKey) + + val eventOwnerHash = + if (event is GiftWrapEvent) { + event.recipientPubKey()?.let { hasher.hash(it) } ?: pubkeyHash + } else { + pubkeyHash + } + + val eTagHash = hasher.hashETag(event.id) + stmt.bindString(1, event.id) stmt.bindString(2, event.pubKey) stmt.bindLong(3, event.createdAt) - stmt.bindLong(4, event.kind.toLong()) + stmt.bindLong(4, kindLong) stmt.bindString(5, OptimizedJsonMapper.toJson(event.tags)) stmt.bindString(6, event.content) stmt.bindString(7, event.sig) if (event is AddressableEvent) { val dTag = event.dTag() stmt.bindString(8, dTag) - stmt.bindLong(9, hasher.hashETag(event.id)) - stmt.bindLong(10, hasher.hashATag(AddressSerializer.assemble(event.kind, event.pubKey, dTag))) + stmt.bindLong(9, eventOwnerHash) + stmt.bindLong(10, eTagHash) + stmt.bindLong(11, hasher.hashATag(AddressSerializer.assemble(event.kind, event.pubKey, dTag))) } else { stmt.bindNull(8) - stmt.bindLong(9, hasher.hashETag(event.id)) - stmt.bindNull(10) + stmt.bindLong(9, eventOwnerHash) + stmt.bindLong(10, eTagHash) + stmt.bindNull(11) } val headerId = stmt.executeInsert() val stmtTags = db.compileStatement(sqlInsertTags) - event.tags.forEach { tag -> - if (tagIndexStrategy.shouldIndex(event.kind, tag)) { - stmtTags.bindLong(1, headerId) - stmtTags.bindLong(2, hasher.hash(tag[0], tag[1])) - stmtTags.executeInsert() + // sorting helps SQLLite by avoiding + // rebalancing the tree every new insert + val indexableTags = ArrayList() + for (idx in event.tags.indices) { + if (tagIndexStrategy.shouldIndex(event.kind, event.tags[idx])) { + indexableTags.add(hasher.hash(event.tags[idx][0], event.tags[idx][1])) } } + indexableTags.sort() + indexableTags.forEach { + stmtTags.bindLong(1, headerId) + stmtTags.bindLong(2, it) + stmtTags.executeInsert() + } return headerId } @@ -171,10 +195,17 @@ class EventIndexesModule( fun planQuery( filter: Filter, hasher: TagNameValueHasher, + db: SQLiteDatabase, ): String { - val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher) ?: return makeEverythingQuery() + val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher) - return makeQueryIn(rowIdSubQuery.sql) + return if (rowIdSubQuery == null) { + val query = makeEverythingQuery() + db.explainQuery(query) + } else { + val query = makeQueryIn(rowIdSubQuery.sql) + db.explainQuery(query, rowIdSubQuery.args.toTypedArray()) + } } fun query( @@ -196,32 +227,31 @@ class EventIndexesModule( onEach: (T) -> Unit, ) { val rowIdSubQuery = prepareRowIDSubQueries(filter, hasher(db)) ?: return db.runQueryEmitting(makeEverythingQuery(), onEach = onEach) - db.runQueryEmitting(makeQueryIn(rowIdSubQuery.sql), rowIdSubQuery.args, onEach) } fun planQuery( filters: List, hasher: TagNameValueHasher, + db: SQLiteDatabase, ): String { - val rowIdSubQueries = filters.mapNotNull { prepareRowIDSubQueries(it, hasher) } - if (rowIdSubQueries.isEmpty()) return makeEverythingQuery() - val unions = rowIdSubQueries.joinToString(" UNION ") { it.sql } - return makeQueryIn(unions) + val rowIdSubQuery = unionSubqueriesIfNeeded(filters, hasher) + + return if (rowIdSubQuery == null) { + val query = makeEverythingQuery() + db.explainQuery(query) + } else { + val query = makeQueryIn(rowIdSubQuery.sql) + db.explainQuery(query, rowIdSubQuery.args.toTypedArray()) + } } fun query( filters: List, db: SQLiteDatabase, ): List { - val rowIdSubQueries = filters.mapNotNull { prepareRowIDSubQueries(it, hasher(db)) } - - if (rowIdSubQueries.isEmpty()) return db.runQuery(makeEverythingQuery()) - - val unions = rowIdSubQueries.joinToString(" UNION ") { it.sql } - val args = rowIdSubQueries.flatMap { it.args } - - return db.runQuery(makeQueryIn(unions), args) + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return db.runQuery(makeEverythingQuery()) + return db.runQuery(makeQueryIn(rowIdSubqueries.sql), rowIdSubqueries.args) } fun query( @@ -229,14 +259,9 @@ class EventIndexesModule( db: SQLiteDatabase, onEach: (T) -> Unit, ) { - val rowIdSubQueries = filters.mapNotNull { prepareRowIDSubQueries(it, hasher(db)) } + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return db.runQueryEmitting(makeEverythingQuery(), onEach = onEach) - if (rowIdSubQueries.isEmpty()) return db.runQueryEmitting(makeEverythingQuery(), onEach = onEach) - - val unions = rowIdSubQueries.joinToString(" UNION ") { it.sql } - val args = rowIdSubQueries.flatMap { it.args } - - db.runQueryEmitting(makeQueryIn(unions), args, onEach) + db.runQueryEmitting(makeQueryIn(rowIdSubqueries.sql), rowIdSubqueries.args, onEach) } private fun makeEverythingQuery() = "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC, id" @@ -244,7 +269,9 @@ class EventIndexesModule( private fun makeQueryIn(rowIdQuery: String) = """ SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN ($rowIdQuery) AS filtered + INNER JOIN ( + $rowIdQuery + ) AS filtered ON event_headers.row_id = filtered.row_id ORDER BY created_at DESC, id """.trimIndent() @@ -320,14 +347,9 @@ class EventIndexesModule( filters: List, db: SQLiteDatabase, ): Int { - val rowIdSubQueries = filters.mapNotNull { prepareRowIDSubQueries(it, hasher(db)) } + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return db.countEverything() - if (rowIdSubQueries.isEmpty()) return db.countEverything() - - val unions = rowIdSubQueries.joinToString(" UNION ") { it.sql } - val args = rowIdSubQueries.flatMap { it.args } - - return db.countIn(unions, args) + return db.countIn(rowIdSubqueries.sql, rowIdSubqueries.args) } private fun SQLiteDatabase.countEverything() = runCount("SELECT count(*) as count FROM event_headers") @@ -361,14 +383,9 @@ class EventIndexesModule( filters: List, db: SQLiteDatabase, ): Int { - val rowIdSubqueries = filters.mapNotNull { prepareRowIDSubQueries(it, hasher(db)) } + val rowIdSubqueries = unionSubqueriesIfNeeded(filters, hasher(db)) ?: return 0 - if (rowIdSubqueries.isEmpty()) return 0 - - val unions = rowIdSubqueries.joinToString(" UNION ") { it.sql } - val args = rowIdSubqueries.flatMap { it.args } - - return db.runDelete(unions, args) + return db.runDelete(rowIdSubqueries.sql, rowIdSubqueries.args) } private fun SQLiteDatabase.runDelete( @@ -376,6 +393,30 @@ class EventIndexesModule( args: List = emptyList(), ): Int = delete("event_headers", "row_id IN ($sql)", args.toTypedArray()) + // --------------------------------- + // Prepare unions of all the filters + // --------------------------------- + fun unionSubqueriesIfNeeded( + filters: List, + hasher: TagNameValueHasher, + ): RowIdSubQuery? { + val inner = + filters.mapNotNull { filter -> + prepareRowIDSubQueries(filter, hasher) + } + + if (inner.isEmpty()) return null + + return if (inner.size == 1) { + inner.first() + } else { + RowIdSubQuery( + sql = inner.joinToString("\n UNION\n ") { "SELECT row_id FROM (${it.sql})" }, + args = inner.flatMap { it.args }, + ) + } + } + // ---------------------------- // Inner row id selections // ---------------------------- @@ -385,92 +426,105 @@ class EventIndexesModule( ): RowIdSubQuery? { if (!filter.isFilledFilter()) return null + val mustJoinSearch = (filter.search != null) + + val nonDTags = filter.tags?.filter { it.key != "d" } ?: emptyMap() + val hasHeaders = with(filter) { - (ids != null && ids.isNotEmpty()) || + (ids != null) || (authors != null && authors.isNotEmpty()) || (kinds != null && kinds.isNotEmpty()) || + (tags != null && tags.containsKey("d")) || (since != null) || (until != null) || - (tags != null && tags.containsKey("d")) + (limit != null) } - val hasSearch = (filter.search != null && filter.search.isNotBlank()) + var defaultTagKey: String? = null val projection = buildString { - val joins = mutableListOf() + // always do tags if there are any + if (nonDTags.isNotEmpty()) { + append("SELECT event_tags.event_header_row_id as row_id FROM event_tags ") - if (hasHeaders) { - append("SELECT event_headers.row_id as row_id FROM event_headers") - - if (hasSearch) { - joins.add("INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_headers.row_id") - } - - filter.tags?.forEach { (tagName, _) -> - if (tagName != "d") { - joins.add("INNER JOIN event_tags as tag$tagName ON tag$tagName.event_header_row_id = event_headers.row_id") + // it's quite rare to have 2 tags in the filter, but possible + nonDTags.keys.forEachIndexed { index, tagName -> + if (index > 0) { + append("INNER JOIN event_tags as event_tags$tagName ON event_tags$tagName.event_header_row_id = event_tags.event_header_row_id ") + } else { + defaultTagKey = tagName } } - } else if (hasSearch) { - append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName}") - filter.tags?.forEach { (tagName, _) -> - if (tagName != "d") { - joins.add("INNER JOIN event_tags as tag$tagName ON tag$tagName.event_header_row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") - } + if (hasHeaders) { + append("INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id ") + } + + if (mustJoinSearch) { + append("INNER JOIN ${fts.tableName} ON ${fts.tableName}.${fts.eventHeaderRowIdName} = event_tags.event_header_row_id ") + } + } else if (mustJoinSearch) { + append("SELECT ${fts.tableName}.${fts.eventHeaderRowIdName} as row_id FROM ${fts.tableName} ") + + if (hasHeaders) { + append("INNER JOIN event_headers ON event_headers.row_id = ${fts.tableName}.${fts.eventHeaderRowIdName}") } } else { - // has only tags - filter.tags?.forEach { (tagName, _) -> - if (tagName != "d") { - if (isEmpty()) { - append("SELECT tag$tagName.event_header_row_id as row_id FROM event_tags as tag$tagName") - } else { - joins.add("INNER JOIN event_tags as tag$tagName ON tag$tagName.event_header_row_id = tag${tagName.takeLast(1)}.event_header_row_id") - } - } - } - - if (isEmpty()) { - // only limit is present - append("SELECT event_headers.row_id as row_id FROM event_headers") - } - } - - if (joins.isNotEmpty()) { - append(" ${joins.joinToString(" ")}") + // no tags and no search. + append("SELECT event_headers.row_id as row_id FROM event_headers ") } } val clause = where { + // the order should match indexes + // ids reduce the filter the most filter.ids?.let { equalsOrIn("event_headers.id", it) } - filter.kinds?.let { equalsOrIn("event_headers.kind", it) } - filter.authors?.let { equalsOrIn("event_headers.pubkey", it) } + + // range search is bad but most of the time these are up the top with few elements. filter.since?.let { greaterThanOrEquals("event_headers.created_at", it) } filter.until?.let { lessThanOrEquals("event_headers.created_at", it) } + // there are indexes for these, starting with tags. + nonDTags.forEach { (tagName, tagValues) -> + val column = + if (defaultTagKey == null || defaultTagKey == tagName) { + "event_tags.tag_hash" + } else { + "event_tags$tagName.tag_hash" + } + + equalsOrIn( + column, + tagValues.map { + hasher.hash(tagName, it) + }, + ) + } + + filter.kinds?.let { equalsOrIn("event_headers.kind", it) } + filter.authors?.let { equalsOrIn("event_headers.pubkey", it) } + + // there are indexes for these, starting with tags. filter.tags?.forEach { (tagName, tagValues) -> if (tagName == "d") { equalsOrIn("event_headers.d_tag", tagValues) - } else { - equalsOrIn( - "tag$tagName.tag_hash", - tagValues.map { - hasher.hash(tagName, it) - }, - ) } } - filter.search?.let { match(fts.tableName, it) } + // if search is included, SQLLite will always start here. + filter.search?.let { + if (it.isNotBlank()) { + match(fts.tableName, it) + } + } } val whereClause = if (filter.limit != null) { - "${clause.conditions} ORDER BY created_at DESC, id ASC LIMIT ${filter.limit}" + "${clause.conditions} ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT ${filter.limit}" } else { clause.conditions } @@ -483,7 +537,7 @@ class EventIndexesModule( db.execSQL("DELETE FROM event_headers") } - class RowIdSubQuery( + data class RowIdSubQuery( val sql: String, val args: List, ) diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationModule.kt index 03f1992f8..2a8b1a94e 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationModule.kt @@ -29,7 +29,7 @@ class ExpirationModule : IModule { db.execSQL( """ CREATE TABLE event_expirations ( - event_header_row_id INTEGER, + event_header_row_id INTEGER PRIMARY KEY NOT NULL, expiration INTEGER NOT NULL, FOREIGN KEY (event_header_row_id) REFERENCES event_headers(row_id) ON DELETE CASCADE ) @@ -49,8 +49,6 @@ class ExpirationModule : IModule { END; """.trimIndent(), ) - - db.execSQL("CREATE UNIQUE INDEX events_exp_id ON event_expirations (event_header_row_id)") } override fun drop(db: SQLiteDatabase) { diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryExplainer.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryExplainer.kt new file mode 100644 index 000000000..68bea3398 --- /dev/null +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryExplainer.kt @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store.sqlite + +import android.database.sqlite.SQLiteDatabase + +fun SQLiteEventStore.explainQuery( + sql: String, + args: Array = emptyArray(), +) = readableDatabase.explainQuery(sql, args.map { it.toString() }.toTypedArray()) + +fun SQLiteDatabase.explainQuery( + sql: String, + args: Array = emptyArray(), +): String = + rawQuery("EXPLAIN QUERY PLAN $sql", args).use { cursor -> + val treeIndex = mutableMapOf() + val rootNodes = mutableListOf() + + while (cursor.moveToNext()) { + val id = cursor.getInt(0) + val parentId = cursor.getInt(1) + val detail = cursor.getString(3) + + val line = PlanNode(detail) + + treeIndex[id] = line + + val parent = treeIndex[parentId] + if (parent != null) { + parent.children.add(line) + } else { + rootNodes.add(line) + } + } + + buildString { + appendLine(populateArgs(sql, args)) + for (idx in rootNodes.indices) { + printNode(rootNodes[idx], "", idx == rootNodes.size - 1, idx > 0) + } + } + } + +private fun populateArgs( + sql: String, + args: Array = emptyArray(), +): String { + var result = sql + args.forEach { + result = result.replaceFirst("?", "\"$it\"") + } + return result +} + +fun StringBuilder.printNode( + node: PlanNode, + prefix: String, + isLast: Boolean, + newLine: Boolean, +) { + if (newLine) append('\n') + append(prefix) + append(if (isLast) "└── " else "├── ") + append(node.detail) + + val newPrefix = prefix + if (isLast) " " else "│ " + for (i in node.children.indices) { + printNode(node.children[i], newPrefix, i == node.children.size - 1, true) + } +} + +data class PlanNode( + val detail: String, + val children: MutableList = mutableListOf(), +) diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableModule.kt index 04a2d3c7f..7aa87e594 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ReplaceableModule.kt @@ -28,7 +28,7 @@ class ReplaceableModule : IModule { """ CREATE UNIQUE INDEX replaceable_idx ON event_headers (kind, pubkey) - WHERE (kind >= 10000 AND kind < 20000) OR (kind IN (0, 3)) + WHERE (kind IN (0, 3)) OR (kind >= 10000 AND kind < 20000) """.trimIndent(), ) @@ -41,14 +41,15 @@ class ReplaceableModule : IModule { CREATE TRIGGER delete_older_replaceable_event BEFORE INSERT ON event_headers FOR EACH ROW - WHEN (NEW.kind >= 10000 AND NEW.kind < 20000) OR (NEW.kind IN (0, 3)) + WHEN (NEW.kind IN (0, 3)) OR (NEW.kind >= 10000 AND NEW.kind < 20000) BEGIN -- Delete older records if this is the newest DELETE FROM event_headers WHERE event_headers.kind = NEW.kind AND event_headers.pubkey = NEW.pubkey AND - event_headers.created_at < NEW.created_at; + event_headers.created_at < NEW.created_at AND + ((event_headers.kind IN (0, 3)) OR (event_headers.kind >= 10000 AND event_headers.kind < 20000)); END; """.trimIndent(), ) diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/RightToVanishModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/RightToVanishModule.kt index db07539c8..979574c10 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/RightToVanishModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/RightToVanishModule.kt @@ -24,20 +24,22 @@ import android.database.sqlite.SQLiteDatabase import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent -class RightToVanishModule : IModule { +class RightToVanishModule( + val hasher: (db: SQLiteDatabase) -> TagNameValueHasher, +) : IModule { override fun create(db: SQLiteDatabase) { db.execSQL( """ CREATE TABLE event_vanish ( - event_header_row_id INTEGER, - pubkey TEXT NOT NULL, + event_header_row_id INTEGER PRIMARY KEY NOT NULL, + pubkey_hash INTEGER NOT NULL, created_at INTEGER NOT NULL, FOREIGN KEY (event_header_row_id) REFERENCES event_headers(row_id) ON DELETE CASCADE ) """.trimIndent(), ) - db.execSQL("CREATE UNIQUE INDEX event_vanish_key ON event_vanish (pubkey)") + db.execSQL("CREATE UNIQUE INDEX event_vanish_key ON event_vanish (pubkey_hash)") db.execSQL( """ @@ -48,8 +50,8 @@ class RightToVanishModule : IModule { -- Delete older records if this is the newest DELETE FROM event_vanish WHERE - event_vanish.created_at < NEW.created_at AND - event_vanish.pubkey = NEW.pubkey; + event_vanish.pubkey_hash = NEW.pubkey_hash AND + event_vanish.created_at < NEW.created_at; END; """.trimIndent(), ) @@ -61,8 +63,8 @@ class RightToVanishModule : IModule { FOR EACH ROW BEGIN DELETE FROM event_headers - WHERE created_at < NEW.created_at AND - pubkey = NEW.pubkey; + WHERE event_headers.created_at < NEW.created_at AND + event_headers.pubkey_owner_hash = NEW.pubkey_hash; END; """.trimIndent(), ) @@ -78,8 +80,8 @@ class RightToVanishModule : IModule { WHERE EXISTS ( SELECT 1 FROM event_vanish WHERE - event_vanish.created_at >= NEW.created_at AND - event_vanish.pubkey = NEW.pubkey + event_vanish.pubkey_hash = NEW.pubkey_owner_hash AND + event_vanish.created_at >= NEW.created_at ); END; """.trimIndent(), @@ -92,7 +94,7 @@ class RightToVanishModule : IModule { val insertRTV = """ - INSERT OR ROLLBACK INTO event_vanish (event_header_row_id, pubkey, created_at) + INSERT OR ROLLBACK INTO event_vanish (event_header_row_id, pubkey_hash, created_at) VALUES (?, ?, ?) """.trimIndent() @@ -105,7 +107,7 @@ class RightToVanishModule : IModule { if (event is RequestToVanishEvent && event.shouldVanishFrom(relayUrl)) { val stmt = db.compileStatement(insertRTV) stmt.bindLong(1, headerId) - stmt.bindString(2, event.pubKey) + stmt.bindLong(2, hasher(db).hash(event.pubKey)) stmt.bindLong(3, event.createdAt) stmt.executeInsert() } diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 3c86d9271..c6b202262 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -32,6 +32,8 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventIndexesModule.IndexingStrategy import com.vitorpamplona.quartz.nip40Expiration.isExpired +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext class SQLiteEventStore( val context: Context, @@ -44,16 +46,17 @@ class SQLiteEventStore( } val seedModule = SeedModule() + val fullTextSearchModule = FullTextSearchModule() - val eventIndexModule = EventIndexesModule(fullTextSearchModule, seedModule, tagIndexStrategy) + val eventIndexModule = EventIndexesModule(fullTextSearchModule, seedModule::hasher, tagIndexStrategy) val replaceableModule = ReplaceableModule() val addressableModule = AddressableModule() val ephemeralModule = EphemeralModule() - val deletionModule = DeletionRequestModule() + val deletionModule = DeletionRequestModule(seedModule::hasher) val expirationModule = ExpirationModule() - val rightToVanishModule = RightToVanishModule() + val rightToVanishModule = RightToVanishModule(seedModule::hasher) val modules = listOf( @@ -112,12 +115,28 @@ class SQLiteEventStore( modules.reversed().forEach { it.deleteAll(db) } } + suspend fun vacuum() { + // 1. ANALYZE: Collects statistics about tables and indices + // to help the query planner optimize queries. + withContext(Dispatchers.IO) { + writableDatabase.execSQL("VACUUM") + } + } + + suspend fun analyse() { + // 2. VACUUM: Rebuilds the database file, reclaiming unused space + // and reducing fragmentation. + withContext(Dispatchers.IO) { + writableDatabase.execSQL("ANALYZE") + } + } + private fun innerInsertEvent( event: Event, db: SQLiteDatabase, ) { val headerId = eventIndexModule.insert(event, db) - deletionModule.insert(event, headerId, db) + deletionModule.insert(event, db) expirationModule.insert(event, headerId, db) fullTextSearchModule.insert(event, headerId, db) rightToVanishModule.insert(event, relayUrl, headerId, db) diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SeedModule.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SeedModule.kt index ece45f6f1..7788213b7 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SeedModule.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SeedModule.kt @@ -76,4 +76,8 @@ class SeedModule : IModule { } override fun deleteAll(db: SQLiteDatabase) {} + + private var hasherCache: TagNameValueHasher? = null + + fun hasher(db: SQLiteDatabase): TagNameValueHasher = hasherCache ?: TagNameValueHasher(getSeed(db)).also { hasherCache = it } } diff --git a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/TagNameValueHasher.kt b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/TagNameValueHasher.kt index 876f26d01..c8b73f1bf 100644 --- a/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/TagNameValueHasher.kt +++ b/quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/TagNameValueHasher.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.quartz.nip01Core.store.sqlite +import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.hints.bloom.MurmurHash3 /** @@ -32,6 +33,9 @@ class TagNameValueHasher( val hasher = MurmurHash3() // small performance improvements on inserting + val pTagHash by lazy { + hasher.hash128x64Half("p".encodeToByteArray(), seed) + } val eTagHash by lazy { hasher.hash128x64Half("e".encodeToByteArray(), seed) } @@ -56,4 +60,8 @@ class TagNameValueHasher( fun hashATag(value: String) = hasher.hash128x64Half(value.encodeToByteArray(), aTagHash) fun hashETag(value: String) = hasher.hash128x64Half(value.encodeToByteArray(), eTagHash) + + fun hashPTag(value: String) = hasher.hash128x64Half(value.encodeToByteArray(), pTagHash) + + fun hash(value: HexKey) = hasher.hash128x64Half(value.encodeToByteArray(), seed) } diff --git a/quartz/src/androidUnitTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqllite/EventDbQueryAssemblerTest.kt b/quartz/src/androidUnitTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqllite/EventDbQueryAssemblerTest.kt deleted file mode 100644 index 8575333c5..000000000 --- a/quartz/src/androidUnitTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqllite/EventDbQueryAssemblerTest.kt +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store.sqlite - -import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import junit.framework.TestCase -import kotlin.test.Test - -class EventDbQueryAssemblerTest { - val builder = EventIndexesModule(FullTextSearchModule(), SeedModule()) - val hasher = TagNameValueHasher(0L) - - val key1 = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d" - val key2 = "f3ac434d61bc0f491a814782ccfdf9c439dae1f0bde9097ad4a245f4c495cd14" - val key3 = "12ae0fd81c85e1e7d9ed096397dc3129849425fe6f8afce7213ebf38ddfc6ca9" - - @Test - fun testEmpty() { - val sql = builder.planQuery(Filter(), hasher) - TestCase.assertEquals( - "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers ORDER BY created_at DESC, id", - sql, - ) - } - - @Test - fun testLimit() { - val sql = builder.planQuery(Filter(limit = 10), hasher) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN (SELECT event_headers.row_id as row_id FROM event_headers WHERE 1 = 1 ORDER BY created_at DESC, id ASC LIMIT 10) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC, id - """.trimIndent(), - sql, - ) - } - - @Test - fun testAllFearures() { - val sql = - builder.planQuery( - listOf( - Filter(limit = 10), - Filter(authors = listOf(key1), kinds = listOf(1, 1111), search = "keywords", limit = 100), - Filter(kinds = listOf(20), search = "cats", limit = 30), - ), - hasher, - ) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN (SELECT event_headers.row_id as row_id FROM event_headers WHERE 1 = 1 ORDER BY created_at DESC, id ASC LIMIT 10 UNION SELECT event_headers.row_id as row_id FROM event_headers INNER JOIN event_fts ON event_fts.event_header_row_id = event_headers.row_id WHERE (event_headers.kind IN (?, ?)) AND (event_headers.pubkey = ?) AND (event_fts MATCH ?) ORDER BY created_at DESC, id ASC LIMIT 100 UNION SELECT event_headers.row_id as row_id FROM event_headers INNER JOIN event_fts ON event_fts.event_header_row_id = event_headers.row_id WHERE (event_headers.kind = ?) AND (event_fts MATCH ?) ORDER BY created_at DESC, id ASC LIMIT 30) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC, id - """.trimIndent(), - sql, - ) - } - - @Test - fun testIdQuery() { - val sql = builder.planQuery(Filter(ids = listOf(key1)), hasher) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN (SELECT event_headers.row_id as row_id FROM event_headers WHERE event_headers.id = ?) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC, id - """.trimIndent(), - sql, - ) - } - - @Test - fun testAuthors() { - val sql = builder.planQuery(Filter(authors = listOf(key1, key2)), hasher) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN (SELECT event_headers.row_id as row_id FROM event_headers WHERE event_headers.pubkey IN (?, ?)) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC, id - """.trimIndent(), - sql, - ) - } - - @Test - fun testAuthorsAndSearch() { - val sql = builder.planQuery(Filter(authors = listOf(key1, key2, key3), search = "keywords"), hasher) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN (SELECT event_headers.row_id as row_id FROM event_headers INNER JOIN event_fts ON event_fts.event_header_row_id = event_headers.row_id WHERE (event_headers.pubkey IN (?, ?, ?)) AND (event_fts MATCH ?)) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC, id - """.trimIndent(), - sql, - ) - } - - @Test - fun testKindAndSearch() { - val sql = builder.planQuery(Filter(kinds = listOf(1, 1111, 10000), search = "keywords"), hasher) - TestCase.assertEquals( - """ - SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers - INNER JOIN (SELECT event_headers.row_id as row_id FROM event_headers INNER JOIN event_fts ON event_fts.event_header_row_id = event_headers.row_id WHERE (event_headers.kind IN (?, ?, ?)) AND (event_fts MATCH ?)) AS filtered - ON event_headers.row_id = filtered.row_id - ORDER BY created_at DESC, id - """.trimIndent(), - sql, - ) - } -} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip59Giftwrap/wraps/GiftWrapEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip59Giftwrap/wraps/GiftWrapEvent.kt index b408fd19b..de404e7a8 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip59Giftwrap/wraps/GiftWrapEvent.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip59Giftwrap/wraps/GiftWrapEvent.kt @@ -26,7 +26,7 @@ import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.firstTagValue import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner -import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync import com.vitorpamplona.quartz.nip01Core.tags.people.PTag import com.vitorpamplona.quartz.nip21UriScheme.toNostrUri import com.vitorpamplona.quartz.nip40Expiration.ExpirationTag @@ -104,13 +104,13 @@ class GiftWrapEvent( const val KIND = 1059 const val ALT = "Encrypted event" - suspend fun create( + fun create( event: Event, recipientPubKey: HexKey, expirationDelta: Long? = null, createdAt: Long = TimeUtils.randomWithTwoDays(), ): GiftWrapEvent { - val signer = NostrSignerInternal(KeyPair()) // GiftWrap is always a random key + val signer = NostrSignerSync(KeyPair()) // GiftWrap is always a random key val tags = expirationDelta?.let {