nick/amethyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Faster event serializer -> sha256 -> id comparison without creating intermediary buffers.

Browse Source
This commit is contained in:
Vitor Pamplona
2025-12-23 09:39:30 -05:00
parent 01ea413afd
commit aa2f969be6
10 changed files with 269 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/src/androidTest/java/com/vitorpamplona/quartz/benchmark/EventVerifyCombined.kt
+61
View File
@@ -0,0 +1,61 @@
/**
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.benchmark
import androidx.benchmark.junit4.BenchmarkRule
import androidx.benchmark.junit4.measureRepeated
import androidx.test.ext.junit.runners.AndroidJUnit4
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.crypto.EventHasher
import com.vitorpamplona.quartz.nip01Core.crypto.EventHasherSerializer
import com.vitorpamplona.quartz.utils.Hex
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
/**
* Benchmark, which will execute on an Android device.
*
* The body of [BenchmarkRule.measureRepeated] is measured in a loop, and Studio will output the
* result. Modify your code to see how it affects performance.
*/
@RunWith(AndroidJUnit4::class)
class EventVerifyCombined {
@get:Rule
val benchmarkRule = BenchmarkRule()
@Test
fun serializeAndCheckId() {
val event = Event.fromJson(largeKind1Event)
benchmarkRule.measureRepeated {
val outId = EventHasher.hashIdBytes(event.pubKey, event.createdAt, event.kind, event.tags, event.content)
Hex.isEqual(event.id, outId)
}
}
@Test
fun fastSerializeAndCheckId() {
val event = Event.fromJson(largeKind1Event)
benchmarkRule.measureRepeated {
EventHasherSerializer.makeJsonForIdHashAndCheck(event.id, event.pubKey, event.createdAt, event.kind, event.tags, event.content)
}
}
}
quartz/src/androidInstrumentedTest/kotlin/com/vitorpamplona/quartz/nip01Core/EventSigCheck.kt
+10
View File
@@ -88,8 +88,10 @@ class EventSigCheck {
val old = EventHasherSerializer.makeJsonForId(event.pubKey, event.createdAt, event.kind, event.tags, event.content)
val new = EventHasherSerializer.fastMakeJsonForId(event.pubKey, event.createdAt, event.kind, event.tags, event.content)
val new2 = EventHasherSerializer.makeJsonForIdHashAndCheck(event.id, event.pubKey, event.createdAt, event.kind, event.tags, event.content)
assertEquals(old.toByteArray().joinToString(), new.joinToString())
assertTrue(new2)
assertTrue(event.verifySignature())
assertTrue(event.verifyId())
}
@@ -122,11 +124,14 @@ class EventSigCheck {
val old = EventHasherSerializer.makeJsonForId(event.pubKey, event.createdAt, event.kind, event.tags, event.content)
val new = EventHasherSerializer.fastMakeJsonForId(event.pubKey, event.createdAt, event.kind, event.tags, event.content)
val new2 = EventHasherSerializer.makeJsonForIdHashAndCheck(event.id, event.pubKey, event.createdAt, event.kind, event.tags, event.content)
println(old)
println(String(new))
println(new2)
assertEquals(old, String(new))
assertTrue(new2)
assertTrue(event.verifySignature())
assertTrue(event.verifyId())
}
@@ -152,13 +157,18 @@ class EventSigCheck {
sig = "81d08765524bd8b774585a57c76d25b1d2c09eaa23efcb075226ba8b64f42e3d9d9403e5edf888e0e58702a24abc084259e21b97e24a275021c5e36186c65f0c",
)
println("TESTING")
val old = EventHasherSerializer.makeJsonForId(event.pubKey, event.createdAt, event.kind, event.tags, event.content)
val new = EventHasherSerializer.fastMakeJsonForId(event.pubKey, event.createdAt, event.kind, event.tags, event.content)
val new2 = EventHasherSerializer.makeJsonForIdHashAndCheck(event.id, event.pubKey, event.createdAt, event.kind, event.tags, event.content)
println(old)
println(String(new))
println(new2)
assertEquals(old.toByteArray().joinToString(), new.joinToString())
assertTrue(new2)
assertTrue(event.verifySignature())
assertTrue(event.verifyId())
}
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/crypto/EventExt.kt
+1 -1
View File
@@ -28,7 +28,7 @@ fun Event.generateId(): String = EventHasher.hashId(pubKey, createdAt, kind, tag
fun Event.verifyId(): Boolean {
if (id.isEmpty()) return false
return EventHasher.hashIdEquals(id, pubKey, createdAt, kind, tags, content)
return EventHasher.hashIdCheck(id, pubKey, createdAt, kind, tags, content)
}
fun Event.verifySignature(): Boolean {
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/crypto/EventHasher.kt
+2 -6
View File
@@ -22,7 +22,6 @@ package com.vitorpamplona.quartz.nip01Core.crypto
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.core.toHexKey
import com.vitorpamplona.quartz.utils.Hex
import com.vitorpamplona.quartz.utils.sha256.sha256
class EventHasher {
@@ -45,16 +44,13 @@ class EventHasher {
content: String,
): String = hashIdBytes(pubKey, createdAt, kind, tags, content).toHexKey()
fun hashIdEquals(
fun hashIdCheck(
id: HexKey,
pubKey: HexKey,
createdAt: Long,
kind: Int,
tags: Array<Array<String>>,
content: String,
): Boolean {
val outId = hashIdBytes(pubKey, createdAt, kind, tags, content)
return Hex.isEqual(id, outId)
}
): Boolean = EventHasherSerializer.makeJsonForIdHashAndCheck(id, pubKey, createdAt, kind, tags, content)
}
}
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/crypto/EventHasherSerializer.kt
+9
View File
@@ -41,4 +41,13 @@ expect object EventHasherSerializer {
tags: Array<Array<String>>,
content: String,
): ByteArray
fun makeJsonForIdHashAndCheck(
id: HexKey,
pubKey: HexKey,
createdAt: Long,
kind: Int,
tags: Array<Array<String>>,
content: String,
): Boolean
}
quartz/src/iosMain/kotlin/com/vitorpamplona/quartz/nip01Core/crypto/EventHasherSerializer.ios.kt
+11
View File
@@ -42,4 +42,15 @@ actual object EventHasherSerializer {
): ByteArray {
TODO("Not yet implemented")
}
actual fun makeJsonForIdHashAndCheck(
id: HexKey,
pubKey: HexKey,
createdAt: Long,
kind: Int,
tags: Array<Array<String>>,
content: String,
): Boolean {
TODO("Not yet implemented")
}
}
quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/crypto/EventHasherSerializer.jvmAndroid.kt
+50
View File
@@ -29,7 +29,10 @@ import com.fasterxml.jackson.databind.JsonMappingException
import com.fasterxml.jackson.databind.node.ArrayNode
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.core.fastForEach
import com.vitorpamplona.quartz.nip01Core.jackson.JacksonMapper
import com.vitorpamplona.quartz.utils.Hex
import com.vitorpamplona.quartz.utils.sha256.pool
import java.io.IOException
actual object EventHasherSerializer {
@@ -111,4 +114,51 @@ actual object EventHasherSerializer {
br.releaseToPool()
}
}
/** reuses the byte buffer straight into Sha256 to avoid creating
* and then sends it to the hex checker to avoid returning
*/
actual fun makeJsonForIdHashAndCheck(
id: HexKey,
pubKey: HexKey,
createdAt: Long,
kind: Int,
tags: Array<Array<String>>,
content: String,
): Boolean {
val br: BufferRecycler = JacksonMapper.mapper.factory._getBufferRecycler()
val bb = HashingByteArrayBuilder(br, pool)
try {
val generator = JacksonMapper.mapper.createGenerator(bb, JsonEncoding.UTF8)
generator.enable(JsonGenerator.Feature.COMBINE_UNICODE_SURROGATES_IN_UTF8)
generator.use {
it.writeStartArray()
it.writeNumber(0)
it.writeString(pubKey)
it.writeNumber(createdAt)
it.writeNumber(kind)
it.writeStartArray()
tags.fastForEach { tag ->
it.writeStartArray()
tag.fastForEach { value ->
it.writeString(value)
}
it.writeEndArray()
}
it.writeEndArray()
it.writeString(content)
it.writeEndArray()
}
return Hex.isEqual(id, bb.sha256())
} catch (e: JsonProcessingException) {
throw e
} catch (e: IOException) {
// shouldn't really happen, but is declared as possibility so:
throw JsonMappingException.fromUnexpectedIOE(e)
} finally {
bb.release()
br.releaseToPool()
}
}
}
quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/crypto/HashingByteArrayBuilder.kt
+106
View File
@@ -0,0 +1,106 @@
/**
* Copyright (c) 2025 Vitor Pamplona
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
* Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
* AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package com.vitorpamplona.quartz.nip01Core.crypto
import com.fasterxml.jackson.core.util.BufferRecycler
import com.vitorpamplona.quartz.utils.sha256.Sha256Pool
import java.io.OutputStream
import kotlin.math.max
import kotlin.math.min
/**
* Computes hashes of the byte array produced by Jackson
* while reusing the largest buffer it ended up being.
*
* This was based on the ByteArrayBuilder form Jackson
*/
class HashingByteArrayBuilder(
private val bufferRecycler: BufferRecycler,
private val hashPool: Sha256Pool,
) : OutputStream() {
var buf: ByteArray = bufferRecycler.allocByteBuffer(BufferRecycler.BYTE_WRITE_CONCAT_BUFFER)
var bufLength: Int = 0
// lazy minimizes lock conflicts with the pool
val hasher by lazy {
hashPool.acquire()
}
fun release() {
bufLength = 0
bufferRecycler.releaseByteBuffer(BufferRecycler.BYTE_WRITE_CONCAT_BUFFER, buf)
hashPool.release(hasher)
buf = EMPTY_ARRAY
}
fun sha256(): ByteArray {
if (bufLength > 0) {
hasher.update(buf, 0, bufLength)
}
return hasher.digest()
}
override fun write(b: ByteArray) = write(b, 0, b.size)
override fun write(
b: ByteArray,
off: Int,
len: Int,
) {
var off = off
var len = len
while (true) {
val toCopy = min(buf.size - bufLength, len)
if (toCopy > 0) {
b.copyInto(buf, bufLength, off, off + toCopy)
off += toCopy
this.bufLength += toCopy
len -= toCopy
}
if (len <= 0) break
allocMore()
}
}
override fun write(b: Int) {
if (this.bufLength >= buf.size) {
allocMore()
}
this.buf[this.bufLength++] = b.toByte()
}
override fun close() {}
override fun flush() {
hasher.update(buf, 0, bufLength)
bufLength = 0
}
private fun allocMore() {
flush()
buf = ByteArray(max((buf.size shr 1), 1000))
}
companion object {
val EMPTY_ARRAY = ByteArray(0)
}
}
quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/sha256/Sha256Hasher.jvmAndroid.kt
+8
View File
@@ -28,8 +28,16 @@ class Sha256Hasher {
fun hash(byteArray: ByteArray) = digest.digest(byteArray).also { digest.reset() }
fun update(
input: ByteArray,
offset: Int,
length: Int,
) = digest.update(input, offset, length)
fun digest(byteArray: ByteArray) = digest.digest(byteArray)
fun digest() = digest.digest()
fun reset() = digest.reset()
/**
quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/sha256/Sha256Pool.jvmAndroid.kt
+11 -2
View File
@@ -35,14 +35,14 @@ class Sha256Pool(
}
}
private fun acquire(): Sha256Hasher {
fun acquire(): Sha256Hasher {
if (pool.isEmpty()) {
Log.w("SHA256Pool", "Pool running low in available digests")
}
return pool.take()
}
private fun release(digest: Sha256Hasher) {
fun release(digest: Sha256Hasher) {
digest.reset()
pool.put(digest)
}
@@ -56,6 +56,15 @@ class Sha256Pool(
}
}
inline fun hasher(action: (hasher: Sha256Hasher) -> ByteArray): ByteArray {
val hasher = acquire()
try {
return action(hasher)
} finally {
release(hasher)
}
}
/**
* Calculate SHA256 hash by streaming the input in chunks.
* This avoids loading the entire input into memory at once.