fix: correct MLS wire format encoding for interop with OpenMLS/mls-rs

Three encoding bugs found by IETF interop test vectors:

1. ExpandWithLabel: label and context length prefixes must use
   QUIC-style variable-length integer encoding (VarInt), not fixed-size
   opaque prefixes. Values < 64 use 1 byte, 64-16383 use 2 bytes with
   0x40 prefix. This is critical when GroupContext (112+ bytes) is
   passed as context.

2. RefHash: label and value also use VarInt-prefixed opaque fields,
   matching the MLS TLS codec convention.

3. SecretTree: DeriveTreeSecret must pass the generation counter as a
   uint32 big-endian context parameter, not empty context. This affects
   key/nonce derivation and ratchet advancement.

Also fixes:
- SignContent and EncryptWithLabel/DecryptWithLabel info encoding
  updated to use VarInt
- KeySchedule test updated to use initial_init_secret from test vector
  (not hardcoded zeros)
- Added putOpaqueVarInt() to TlsWriter for QUIC-style VarInt encoding

https://claude.ai/code/session_01NocQDWj2Y92FugjfgazzL3
This commit is contained in:
Claude
2026-04-03 19:42:27 +00:00
parent fe505c2fb9
commit d1409c84bc
5 changed files with 62 additions and 27 deletions
@@ -21,7 +21,6 @@
package com.vitorpamplona.quartz.marmot.mls.interop
import com.vitorpamplona.quartz.TestResourceLoader
import com.vitorpamplona.quartz.marmot.mls.crypto.MlsCryptoProvider
import com.vitorpamplona.quartz.marmot.mls.schedule.KeySchedule
import com.vitorpamplona.quartz.nip01Core.core.JsonMapper
import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray
@@ -51,8 +50,8 @@ class KeyScheduleInteropTest {
assertTrue(vectors.isNotEmpty(), "No cipher_suite==1 key-schedule vectors found")
for (v in vectors) {
// For the first epoch, init_secret is all zeros
var initSecret = ByteArray(MlsCryptoProvider.HASH_OUTPUT_LENGTH)
// Use the initial_init_secret from the test vector
var initSecret = v.initialInitSecret.hexToByteArray()
for ((epochIdx, epoch) in v.epochs.withIndex()) {
val groupContext = epoch.groupContext.hexToByteArray()
@@ -129,7 +128,7 @@ class KeyScheduleInteropTest {
assertTrue(vectors.isNotEmpty(), "No cipher_suite==1 key-schedule vectors found")
for (v in vectors) {
var initSecret = ByteArray(MlsCryptoProvider.HASH_OUTPUT_LENGTH)
var initSecret = v.initialInitSecret.hexToByteArray()
for ((epochIdx, epoch) in v.epochs.withIndex()) {
val groupContext = epoch.groupContext.hexToByteArray()
@@ -106,6 +106,8 @@ data class TreeMathVector(
@Serializable
data class KeyScheduleVector(
@SerialName("cipher_suite") val cipherSuite: Int,
@SerialName("group_id") val groupId: String,
@SerialName("initial_init_secret") val initialInitSecret: String,
val epochs: List<KeyScheduleEpoch>,
)