nick/amethyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Adds created_at to tags to speed up tag-ordered queries

Browse Source
This commit is contained in:
Vitor Pamplona
2025-12-30 16:23:11 -05:00
parent 5eb9d230e2
commit 7b696f5f54
4 changed files with 72 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionTest.kt
+2 -2
View File
@@ -227,8 +227,8 @@ class DeletionTest {
ON event_headers.row_id = event_tags.event_header_row_id
WHERE
event_tags.tag_hash IN (3221122, 223322) AND
event_tags.created_at >= 1766686500 AND
event_headers.kind = 5 AND
event_headers.created_at >= 1766686500 AND
event_headers.pubkey_owner_hash = 22332323
""".trimIndent()
@@ -238,7 +238,7 @@ class DeletionTest {
TestCase.assertEquals(
"""
${sql.replace("\n","\n ")}
├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?)
├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=? AND created_at>?)
└── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
""".trimIndent(),
explainer,
quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/QueryAssemblerTest.kt
+25 -24
View File
@@ -31,7 +31,7 @@ import org.junit.Before
import org.junit.Test
class QueryAssemblerTest {
val hasher = TagNameValueHasher(0L)
val hasher = TagNameValueHasher(0)
val builder = EventIndexesModule(FullTextSearchModule(), { hasher })
val key1 = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d"
@@ -81,13 +81,14 @@ class QueryAssemblerTest {
"""
SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers
INNER JOIN (
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_headers.created_at >= "1750889190") AND (event_tags.tag_hash = "2657743813502222172") AND (event_headers.kind = "5") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d")
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "2657743813502222172") AND (event_tags.created_at >= "1750889190") AND (event_headers.kind = "5") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d")
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY created_at DESC, id
├── CO-ROUTINE filtered
│ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?)
│ └── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=? AND created_at>?)
│ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ └── USE TEMP B-TREE FOR DISTINCT
├── SCAN filtered
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
@@ -278,14 +279,14 @@ class QueryAssemblerTest {
"""
SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers
INNER JOIN (
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_tags.created_at DESC LIMIT 30
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY created_at DESC, id
├── CO-ROUTINE filtered
│ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ └── USE TEMP B-TREE FOR ORDER BY
│ └── USE TEMP B-TREE FOR DISTINCT
├── SCAN filtered
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
@@ -309,14 +310,13 @@ class QueryAssemblerTest {
"""
SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers
INNER JOIN (
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE event_tags.tag_hash = "-4551135004136952885" ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags WHERE event_tags.tag_hash = "-4551135004136952885" ORDER BY event_tags.created_at DESC LIMIT 30
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY created_at DESC, id
├── CO-ROUTINE filtered
│ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ └── USE TEMP B-TREE FOR ORDER BY
│ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?)
│ └── USE TEMP B-TREE FOR DISTINCT
├── SCAN filtered
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
@@ -344,14 +344,14 @@ class QueryAssemblerTest {
"""
SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers
INNER JOIN (
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.kind = "3") ORDER BY event_tags.created_at DESC LIMIT 30
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY created_at DESC, id
├── CO-ROUTINE filtered
│ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ └── USE TEMP B-TREE FOR ORDER BY
│ └── USE TEMP B-TREE FOR DISTINCT
├── SCAN filtered
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
@@ -376,14 +376,14 @@ class QueryAssemblerTest {
"""
SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers
INNER JOIN (
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_headers.pubkey = "7c5eb72a4584fdaaeaa145b25c92ea9917704224951219dbd43acef9e91fb88d") ORDER BY event_tags.created_at DESC LIMIT 30
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY created_at DESC, id
├── CO-ROUTINE filtered
│ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ └── USE TEMP B-TREE FOR ORDER BY
│ └── USE TEMP B-TREE FOR DISTINCT
├── SCAN filtered
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
@@ -412,15 +412,15 @@ class QueryAssemblerTest {
"""
SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers
INNER JOIN (
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsIn1 ON event_tagsIn1.event_header_row_id = event_tags.event_header_row_id INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tagsIn1.tag_hash = "-6379614208644810021") AND (event_headers.kind = "1") ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT 30
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsIn1 ON event_tagsIn1.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn1.created_at = event_tags.created_at INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "-4551135004136952885") AND (event_tagsIn1.tag_hash = "-6379614208644810021") AND (event_headers.kind = "1") ORDER BY event_tags.created_at DESC LIMIT 30
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY created_at DESC, id
├── CO-ROUTINE filtered
│ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
│ ├── SEARCH event_tagsIn1 USING COVERING INDEX query_by_tags_hash (tag_hash=? AND event_header_row_id=?)
│ └── USE TEMP B-TREE FOR ORDER BY
│ ├── SEARCH event_tagsIn1 USING INDEX query_by_tags_hash (tag_hash=? AND event_header_row_id=?)
│ └── USE TEMP B-TREE FOR DISTINCT
├── SCAN filtered
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
@@ -517,13 +517,14 @@ class QueryAssemblerTest {
"""
SELECT id, pubkey, created_at, kind, tags, content, sig FROM event_headers
INNER JOIN (
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsAll0_1 ON event_tagsAll0_1.event_header_row_id = event_tags.event_header_row_id WHERE (event_tags.tag_hash = "884286737453847614") AND (event_tagsAll0_1.tag_hash = "-4988851810256311323")
SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags INNER JOIN event_tags as event_tagsAll0_1 ON event_tagsAll0_1.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll0_1.created_at = event_tags.created_at WHERE (event_tags.tag_hash = "884286737453847614") AND (event_tagsAll0_1.tag_hash = "-4988851810256311323")
) AS filtered
ON event_headers.row_id = filtered.row_id
ORDER BY created_at DESC, id
├── CO-ROUTINE filtered
│ ├── SEARCH event_tags USING COVERING INDEX query_by_tags_hash (tag_hash=?)
│ └── SEARCH event_tagsAll0_1 USING COVERING INDEX query_by_tags_hash (tag_hash=? AND event_header_row_id=?)
│ ├── SEARCH event_tags USING INDEX query_by_tags_hash (tag_hash=?)
│ ├── SEARCH event_tagsAll0_1 USING INDEX query_by_tags_hash (tag_hash=? AND created_at=?)
│ └── USE TEMP B-TREE FOR DISTINCT
├── SCAN filtered
├── SEARCH event_headers USING INTEGER PRIMARY KEY (rowid=?)
└── USE TEMP B-TREE FOR ORDER BY
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/DeletionRequestModule.kt
+7 -3
View File
@@ -43,7 +43,11 @@ class DeletionRequestModule(
BEFORE INSERT ON event_headers
FOR EACH ROW
BEGIN
-- Check for ID-based deletion record
-- Checks if this event hasn't already been deleted. If so, reject it.
-- Highly optimized using hash + created_at (duplicated in tags) index:
---- Expects tag_hash to be non-existant for new events (quick exit)
---- Expects created-at >= event's to not exist (quick exit)
SELECT RAISE(ABORT, 'blocked: a deletion event exists')
WHERE EXISTS (
SELECT 1 FROM event_tags
@@ -51,9 +55,9 @@ class DeletionRequestModule(
ON event_headers.row_id = event_tags.event_header_row_id
WHERE
event_tags.tag_hash IN (NEW.etag_hash, NEW.atag_hash) AND
event_tags.created_at >= NEW.created_at AND
event_headers.kind = 5 AND
event_headers.pubkey_owner_hash = NEW.pubkey_owner_hash AND
event_headers.created_at >= NEW.created_at
event_headers.pubkey_owner_hash = NEW.pubkey_owner_hash
);
END;
""".trimIndent(),
quartz/src/androidMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt
+38 -14
View File
@@ -63,6 +63,7 @@ class EventIndexesModule(
CREATE TABLE event_tags (
event_header_row_id INTEGER NOT NULL,
tag_hash INTEGER NOT NULL,
created_at INTEGER NOT NULL,
FOREIGN KEY (event_header_row_id) REFERENCES event_headers(row_id) ON DELETE CASCADE
)
""".trimIndent(),
@@ -70,12 +71,15 @@ class EventIndexesModule(
db.execSQL("CREATE UNIQUE INDEX event_headers_id ON event_headers (id)")
db.execSQL("CREATE INDEX query_by_kind_pubkey_dtag_idx ON event_headers (kind, pubkey, d_tag)")
db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers (created_at desc, id)")
db.execSQL("CREATE INDEX query_by_created_at_id ON event_headers (created_at DESC, id)")
// need to check if this is actually needed.
db.execSQL("CREATE INDEX query_by_created_at_kind_key ON event_headers (created_at desc, kind, pubkey)")
db.execSQL("CREATE INDEX query_by_created_at_kind_key ON event_headers (created_at DESC, kind, pubkey)")
db.execSQL("CREATE INDEX fk_event_tags_header_id ON event_tags (event_header_row_id)")
db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, event_header_row_id)")
// This is a very slow index to build (half the insert time goes here) but it is extremely effective.
db.execSQL("CREATE INDEX query_by_tags_hash ON event_tags (tag_hash, created_at DESC)")
// Prevent updates to maintain immutability
db.execSQL(
@@ -117,9 +121,9 @@ class EventIndexesModule(
val sqlInsertTags =
"""
INSERT OR ROLLBACK INTO event_tags
(event_header_row_id, tag_hash)
(event_header_row_id, tag_hash, created_at)
VALUES
(?,?)
(?,?,?)
""".trimIndent()
fun insert(
@@ -177,6 +181,7 @@ class EventIndexesModule(
indexableTags.forEach {
stmtTags.bindLong(1, headerId)
stmtTags.bindLong(2, it)
stmtTags.bindLong(3, event.createdAt)
stmtTags.executeInsert()
}
@@ -462,6 +467,16 @@ class EventIndexesModule(
val nonDTagsAll = filter.tagsAll?.filter { it.key != "d" } ?: emptyMap()
val reverseLookup = nonDTagsIn.isNotEmpty() || nonDTagsAll.isNotEmpty()
val needHeaders =
with(filter) {
(ids != null) ||
(authors != null && authors.isNotEmpty()) ||
(kinds != null && kinds.isNotEmpty()) ||
(tags != null && tags.containsKey("d"))
}
val hasHeaders =
with(filter) {
(ids != null) ||
@@ -478,13 +493,13 @@ class EventIndexesModule(
val projection =
buildString {
// always do tags if there are any
if (nonDTagsIn.isNotEmpty() || nonDTagsAll.isNotEmpty()) {
if (reverseLookup) {
append("SELECT DISTINCT(event_tags.event_header_row_id) as row_id FROM event_tags ")
// it's quite rare to have 2 tags in the filter, but possible
nonDTagsIn.keys.forEachIndexed { index, tagName ->
if (defaultTagKey != null) {
append("INNER JOIN event_tags as event_tagsIn$index ON event_tagsIn$index.event_header_row_id = event_tags.event_header_row_id ")
append("INNER JOIN event_tags as event_tagsIn$index ON event_tagsIn$index.event_header_row_id = event_tags.event_header_row_id AND event_tagsIn$index.created_at = event_tags.created_at ")
} else {
defaultTagKey = TagNameForQuery.InTags(tagName)
}
@@ -493,14 +508,14 @@ class EventIndexesModule(
nonDTagsAll.keys.forEachIndexed { index, tagName ->
nonDTagsAll[tagName]!!.forEachIndexed { valueIndex, tagValue ->
if (defaultTagKey != null) {
append("INNER JOIN event_tags as event_tagsAll${index}_$valueIndex ON event_tagsAll${index}_$valueIndex.event_header_row_id = event_tags.event_header_row_id ")
append("INNER JOIN event_tags as event_tagsAll${index}_$valueIndex ON event_tagsAll${index}_$valueIndex.event_header_row_id = event_tags.event_header_row_id AND event_tagsAll${index}_$valueIndex.created_at = event_tags.created_at ")
} else {
defaultTagKey = TagNameForQuery.AllTags(tagName, valueIndex)
}
}
}
if (hasHeaders) {
if (needHeaders) {
append("INNER JOIN event_headers ON event_headers.row_id = event_tags.event_header_row_id ")
}
@@ -525,10 +540,6 @@ class EventIndexesModule(
// ids reduce the filter the most
filter.ids?.let { equalsOrIn("event_headers.id", it) }
// range search is bad but most of the time these are up the top with few elements.
filter.since?.let { greaterThanOrEquals("event_headers.created_at", it) }
filter.until?.let { lessThanOrEquals("event_headers.created_at", it) }
// it's quite rare to have 2 tags in the filter, but possible
nonDTagsIn.keys.forEachIndexed { index, tagName ->
val column =
@@ -560,6 +571,15 @@ class EventIndexesModule(
}
}
// range search is bad but most of the time these are up the top with few elements.
if (reverseLookup) {
filter.since?.let { greaterThanOrEquals("event_tags.created_at", it) }
filter.until?.let { lessThanOrEquals("event_tags.created_at", it) }
} else {
filter.since?.let { greaterThanOrEquals("event_headers.created_at", it) }
filter.until?.let { lessThanOrEquals("event_headers.created_at", it) }
}
filter.kinds?.let { equalsOrIn("event_headers.kind", it) }
filter.authors?.let { equalsOrIn("event_headers.pubkey", it) }
@@ -580,7 +600,11 @@ class EventIndexesModule(
val whereClause =
if (filter.limit != null) {
"${clause.conditions} ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT ${filter.limit}"
if (reverseLookup) {
"${clause.conditions} ORDER BY event_tags.created_at DESC LIMIT ${filter.limit}"
} else {
"${clause.conditions} ORDER BY event_headers.created_at DESC, event_headers.id ASC LIMIT ${filter.limit}"
}
} else {
clause.conditions
}