From 8dd98b6b5bedd69f4ff584cc0f960e552c2a1856 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Apr 2026 04:55:34 +0000 Subject: [PATCH] feat(quartz): tag-index uses raw values when fs-safe, _h_ otherwise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit idx/tag// used to bucket every tag value under a 16-hex Murmur hash, so `ls idx/tag/p/` showed opaque hex even though pubkey values are perfectly safe filenames. Now the directory name is: - the raw tag value, when it's filesystem-safe across Linux/macOS/ Windows: 1..180 bytes, printable ASCII (0x21..0x7e), none of `/ \ : * ? " < > |`, no leading dot or `_h_`, no trailing dot/space; - otherwise `_h_`. The `_h_` sentinel can never collide with a raw value (raw values are forbidden from starting with it), so both forms safely live in the same parent dir. Common cases keep their raw form and become directly inspectable: ls idx/tag/p// — every event that p-tagged you ls idx/tag/e// — replies / reactions to an event ls idx/tag/t/nostr/ — every kind-1 with #nostr ls idx/tag/k/30023/ — k-tag pointers to articles ls idx/tag/g/drt3n/ — geohash mentions Routes through the _h_ bucket: emojis, URLs (slashes), `a`-tags (colons), free-form `alt` text, anything ≥ 180 bytes, anything that breaks Windows reserved-name rules. The hash is still seed-salted Murmur64 (parity with the previous bucket), so collisions are caught by FilterMatcher post-filter just like before. Writer (FsIndexer.pathsFor) and reader (FsQueryPlanner.firstTagKey) both route through FsLayout.tagValueDirName, so they always agree. Tests: 5 new in FsQueryTest covering raw ASCII tags landing under named dirs, p-tag pubkeys keeping their raw form, emoji and URL tags falling back to _h_, and round-trip queries hitting both buckets correctly. 122 fs tests green. No migration: existing stores must `amy store scrub` once after the upgrade — old purely-hashed tag dirs become unreachable, and scrub rebuilds idx/ from canonicals using the new naming. --- .../quartz/nip01Core/store/fs/FsIndexer.kt | 2 +- .../quartz/nip01Core/store/fs/FsLayout.kt | 80 +++++++++++++- .../nip01Core/store/fs/FsQueryPlanner.kt | 2 +- .../quartz/nip01Core/store/fs/README.md | 22 +++- .../quartz/nip01Core/store/fs/FsQueryTest.kt | 104 ++++++++++++++++++ 5 files changed, 199 insertions(+), 11 deletions(-) diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt index 71558fa75..0fbb233c3 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsIndexer.kt @@ -62,7 +62,7 @@ internal class FsIndexer( for (tag in event.tags) { if (!indexingStrategy.shouldIndex(event.kind, tag)) continue val h = hasher.hash(tag[0], tag[1]) - out.add(layout.tagEntry(tag[0], h, event.createdAt, event.id)) + out.add(layout.tagEntry(tag[0], tag[1], h, event.createdAt, event.id)) } val exp = event.expiration() if (exp != null && exp > 0) { diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt index 1e4c65007..0e8a5c357 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsLayout.kt @@ -78,12 +78,25 @@ internal class FsLayout( id: HexKey, ): Path = idxOwner.resolve(hashHex(ownerHash)).resolve(entryName(ts, id)) + /** + * Tag-value index entry path. The directory name is the raw tag + * value when it's filesystem-safe (alphanumeric ASCII, no path + * separators or shell-hostile chars, ≤180 bytes); otherwise it's + * `_h_` so emojis, URLs, free text, and overly long values + * still round-trip without breaking on case-insensitive filesystems + * or running into the 255-byte path-component limit. + * + * Common cases — pubkey p-tags, event e-tags, ASCII hashtags, kind + * numbers, geohashes — keep their raw values, so `ls idx/tag/p/ + * /` works directly. + */ fun tagEntry( name: String, + value: String, valueHash: Long, ts: Long, id: HexKey, - ): Path = idxTag.resolve(name).resolve(hashHex(valueHash)).resolve(entryName(ts, id)) + ): Path = idxTag.resolve(name).resolve(tagValueDirName(value, valueHash)).resolve(entryName(ts, id)) /** NIP-40 expiration index entry. `exp` is unix seconds, padded for sort order. */ fun expirationEntry( @@ -103,11 +116,15 @@ internal class FsLayout( /** Directory that holds every indexed value for a tag name. */ fun tagDir(name: String): Path = idxTag.resolve(name) - /** Directory that holds every entry for a specific (tag name, value hash). */ + /** + * Directory that holds every entry for a specific (tag name, value). + * Uses the same raw-when-safe / hashed-otherwise rule as [tagEntry]. + */ fun tagValueDir( name: String, + value: String, valueHash: Long, - ): Path = idxTag.resolve(name).resolve(hashHex(valueHash)) + ): Path = idxTag.resolve(name).resolve(tagValueDirName(value, valueHash)) fun kindDir(kind: Kind): Path = idxKind.resolve(kind.toString()) @@ -220,6 +237,63 @@ internal class FsLayout( */ fun sha256Hex(s: String): String = Hex.encode(sha256(s.encodeToByteArray())) + /** + * Sentinel prefix for hashed tag-value directory names. No + * filesystem-safe raw value can begin with this prefix + * ([isFsSafeTagValue] excludes it), so raw and hashed directory + * names are guaranteed disjoint. + */ + const val HASH_PREFIX = "_h_" + + /** Max raw tag-value length before we hash. 180 leaves headroom for the inner `-` filename inside the 255-byte path-component cap on ext4 / NTFS / APFS. */ + const val TAG_VALUE_RAW_MAX_LEN = 180 + + /** + * Pick a directory name for an indexed tag value. Returns the + * raw value when [isFsSafeTagValue] passes; otherwise + * `HASH_PREFIX + hashHex(murmur)`. Both forms live in the same + * parent dir — they cannot collide because no raw value starts + * with `_h_`. + */ + fun tagValueDirName( + value: String, + valueHash: Long, + ): String = + if (isFsSafeTagValue(value)) { + value + } else { + HASH_PREFIX + hashHex(valueHash) + } + + /** + * Tag value safe to use as a filesystem directory name across + * Linux / macOS / Windows. Constraints: + * + * - length 1..[TAG_VALUE_RAW_MAX_LEN] bytes, + * - every char in printable ASCII (`0x21..0x7E`), + * - none of `/ \ : * ? " < > |` (Windows + path separators), + * - doesn't start with `.` (dotfiles, Windows reserved names), + * - doesn't start with the [HASH_PREFIX] sentinel, + * - doesn't end with `.` or space (Windows truncates these). + * + * Catches the common cases (pubkeys, event ids, ASCII hashtags, + * geohashes, kind numbers) while pushing emoji, URLs, free text + * and overly long values through the hash bucket. + */ + fun isFsSafeTagValue(value: String): Boolean { + if (value.isEmpty() || value.length > TAG_VALUE_RAW_MAX_LEN) return false + if (value.startsWith('.') || value.startsWith(HASH_PREFIX)) return false + val last = value.last() + if (last == '.' || last == ' ') return false + for (ch in value) { + if (ch.code !in 0x21..0x7E) return false + if (ch in UNSAFE_TAG_VALUE_CHARS) return false + } + return true + } + + private val UNSAFE_TAG_VALUE_CHARS = setOf('/', '\\', ':', '*', '?', '"', '<', '>', '|') + /** zero-padded to 10 digits so lex order == chronological order through year 2286. */ fun tsPad(ts: Long): String = ts.toString().padStart(TS_PAD_WIDTH, '0') diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt index 7f0ad8665..1c64f5036 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryPlanner.kt @@ -75,7 +75,7 @@ internal class FsQueryPlanner( } firstTagKey(filter)?.let { (name, values) -> - return mergeDesc(values.map { v -> walkDir(layout.tagValueDir(name, hasher.hash(name, v))) }) + return mergeDesc(values.map { v -> walkDir(layout.tagValueDir(name, v, hasher.hash(name, v))) }) } filter.kinds?.let { kinds -> diff --git a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/README.md b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/README.md index 140d59c9d..b15074ed3 100644 --- a/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/README.md +++ b/quartz/src/jvmMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/README.md @@ -66,7 +66,7 @@ streams and asserts query results agree. │ ├── kind//- # hardlink → events/.../.json │ ├── author//- # hardlink │ ├── owner//- # hardlink; recipient for GiftWrap -│ ├── tag///- # hardlink; single-letter tag names only +│ ├── tag///- # hardlink; single-letter tag names only │ ├── expires_at/- # hardlink; for NIP-40 sweep │ └── fts//- # hardlink; for NIP-50 search │ @@ -89,11 +89,21 @@ streams and asserts query results agree. - **Index entry filenames**: `-` where `` is the event's `created_at` left-padded to 10 digits. Lexicographic sort of a directory listing == chronological sort. -- **Hashes**: - - `sha256Hex(d-tag)` — collision-resistant, used wherever the path - IS the uniqueness constraint (addressable slots, address tombstones). - - `hashHex(murmurLong)` — 16 hex chars, used for tag-value indexes - where occasional collisions are caught by the post-filter. +- **Hashes** (only when needed): + - `sha256Hex(d-tag)` — collision-resistant, always used wherever the + path IS the uniqueness constraint (addressable slots, address + tombstones). + - `idx/tag///` — for the tag indexes the + directory name is the **raw tag value** when filesystem-safe + (printable ASCII, no path separators or shell-hostile chars, + ≤180 bytes). Otherwise it falls back to `_h_` + (16 hex chars after the sentinel). Pubkey p-tags, hex e-tags, + ASCII hashtags, kind numbers, and geohashes all keep their raw + values — `ls idx/tag/p//` works directly. Emojis, + URLs, free-form `alt` text, and `:`-bearing `a`-tags route through + the `_h_` bucket so the filesystem stays portable. + - `hashHex(murmurLong)` — 16 hex chars, used for owner-hash dirs + and (after the `_h_` prefix) the hash bucket inside tag indexes. - **mtime**: every canonical write does `Files.setLastModifiedTime(canonical, FileTime.from(event.createdAt, SECONDS))` so `ls -t`, `find -newermt`, `rsync -a` all behave naturally. diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryTest.kt index 4ca0e9cb0..42989502a 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/fs/FsQueryTest.kt @@ -203,6 +203,110 @@ class FsQueryTest { assertEquals(listOf(both.id), got.map { it.id }) } + // ------------------------------------------------------------------ + // Tag-value directory naming: raw when fs-safe, _h_ otherwise. + // ------------------------------------------------------------------ + + @Test + fun `safe ASCII tag values get raw directory names`() { + val e = + signerA.sign( + createdAt = 10, + kind = 1, + tags = arrayOf(arrayOf("t", "nostr")), + content = "x", + ) + store.insert(e) + // The raw value is the directory name — directly inspectable. + val rawDir = root.resolve("idx/tag/t/nostr") + assertTrue(rawDir.exists(), "ASCII-safe tag should land in idx/tag/t/nostr/") + assertEquals(1, rawDir.listDirectoryEntries().size) + } + + @Test + fun `pubkey p-tag uses raw 64-hex directory name`() { + // The motivating case: notifications. p-tags pointing at a + // pubkey land under idx/tag/p// — no hash, directly + // ls-able. + val target = signerB.pubKey + val e = + signerA.sign( + createdAt = 5, + kind = 1, + tags = arrayOf(arrayOf("p", target)), + content = "@you", + ) + store.insert(e) + val pDir = root.resolve("idx/tag/p/$target") + assertTrue(pDir.exists(), "p-tag pubkey should be ls-able directly: idx/tag/p/$target/") + assertEquals(1, pDir.listDirectoryEntries().size) + } + + @Test + fun `tag value with emoji falls back to hashed directory name`() { + val e = + signerA.sign( + createdAt = 10, + kind = 1, + tags = arrayOf(arrayOf("t", "🔥")), + content = "x", + ) + store.insert(e) + // Exactly one entry under t/ and it must be in the _h_ hash + // bucket — emoji is not fs-safe. + val tDir = root.resolve("idx/tag/t") + val entries = tDir.listDirectoryEntries().map { it.fileName.toString() } + assertEquals(1, entries.size, "expected one bucket dir, got: $entries") + assertTrue(entries.single().startsWith("_h_"), "emoji tag must hash; got '${entries.single()}'") + } + + @Test + fun `tag value containing a slash falls back to hashed directory name`() { + val e = + signerA.sign( + createdAt = 10, + kind = 1, + tags = arrayOf(arrayOf("r", "https://example.com/page")), + content = "x", + ) + store.insert(e) + val rDir = root.resolve("idx/tag/r") + val entries = rDir.listDirectoryEntries().map { it.fileName.toString() } + assertEquals(1, entries.size, "expected one bucket dir, got: $entries") + assertTrue(entries.single().startsWith("_h_"), "URL tag must hash; got '${entries.single()}'") + } + + @Test + fun `query round-trips for both raw and hashed values`() { + // Each query must use the same naming rule as the writer or it + // walks a directory that doesn't exist. Insert both a raw-safe + // and a hash-required tag and verify they're both findable. + val safe = + signerA.sign( + createdAt = 1, + kind = 1, + tags = arrayOf(arrayOf("t", "nostr")), + content = "safe", + ) + val unsafe = + signerA.sign( + createdAt = 2, + kind = 1, + tags = arrayOf(arrayOf("t", "🔥")), + content = "unsafe", + ) + store.insert(safe) + store.insert(unsafe) + assertEquals( + listOf(safe.id), + store.query(Filter(tags = mapOf("t" to listOf("nostr")))).map { it.id }, + ) + assertEquals( + listOf(unsafe.id), + store.query(Filter(tags = mapOf("t" to listOf("🔥")))).map { it.id }, + ) + } + @Test fun `non-single-letter tags are not reverse-indexed`() { // SQLite parity: DefaultIndexingStrategy only indexes single-letter