fix(quic): omit IP-literal SNI, bound the handshake, fix dev harness host

Three findings from the local nests interop hang (relay logged "Illegal SNI extension: ignoring IP address presented as hostname"): - TlsClientHello sent the connect host verbatim as TLS SNI, including IP literals. RFC 6066 §3 forbids IP literals in server_name; a strict relay (rustls) discards the extension, ends up with no SNI to select its certificate on, and the handshake stalls. Both ClientHello builders now omit server_name when the host is an IP literal (new isIpLiteralHostname predicate). - QuicWebTransportFactory.connect never bounded awaitHandshake() — only readConnectResponse had a timeout — so a stalled handshake hung connect() indefinitely. It now fails fast after connectTimeoutMillis with a message naming the likely cause. - NostrNestsHarness pointed moqEndpoint at 127.0.0.1, which forced the IP-literal SNI path against the dev relay. Switched to localhost: a valid hostname that matches the relay's generated dev cert and still resolves to loopback. https://claude.ai/code/session_01PoQupfdoKU3ryQwLwTeXeM
2026-05-14 21:41:07 +00:00
parent 806d357569
commit ed1f10c071
4 changed files with 78 additions and 18 deletions
@@ -122,8 +122,16 @@ class QuicWebTransportFactory(
        val driver = QuicConnectionDriver(conn, socket, parentScope)
        driver.start()

+        val handshakeCompleted =
            try {
+                // Bound the handshake — a server-side TLS failure (e.g. a relay
+                // that rejects an IP-literal SNI and can't select a cert) leaves
+                // the handshake stalled with no alert, so without this ceiling
+                // connect() hangs indefinitely instead of failing fast.
+                kotlinx.coroutines.withTimeoutOrNull(connectTimeoutMillis) {
                    conn.awaitHandshake()
+                    true
+                }
            } catch (t: Throwable) {
                driver.close()
                throw WebTransportException(
@@ -132,6 +140,15 @@ class QuicWebTransportFactory(
                    cause = t,
                )
            }
+        if (handshakeCompleted == null) {
+            driver.close()
+            throw WebTransportException(
+                kind = WebTransportException.Kind.HandshakeFailed,
+                message =
+                    "QUIC handshake stalled — no completion within ${connectTimeoutMillis}ms " +
+                        "(a server-side TLS failure such as a rejected IP-literal SNI hangs here with no alert)",
+            )
+        }
        if (conn.status != QuicConnection.Status.CONNECTED) {
            driver.close()
            throw WebTransportException(
@@ -205,7 +205,14 @@ class NostrNestsHarness private constructor(
                // The path under this base is the namespace literal (per
                // moq-rs `claims.root` matching); the `:nestsClient` connect
                // helpers append `/<namespace>?jwt=<token>` themselves.
-                moqEndpoint = "https://127.0.0.1:$MOQ_HOST_PORT/",
+                //
+                // Host MUST be `localhost`, not `127.0.0.1`: the QUIC client
+                // sends the host as TLS SNI, RFC 6066 §3 forbids IP literals
+                // there, and a strict relay (rustls) rejects an IP-literal SNI
+                // and then has no SNI to select its dev cert on — the handshake
+                // stalls. `localhost` is a valid hostname and matches the dev
+                // cert moq-relay generates. Resolves to 127.0.0.1 regardless.
+                moqEndpoint = "https://localhost:$MOQ_HOST_PORT/",
            )
        }

@@ -242,7 +249,9 @@ class NostrNestsHarness private constructor(
            return NostrNestsHarness(
                workDir = null,
                authBaseUrl = "http://127.0.0.1:$AUTH_HOST_PORT",
-                moqEndpoint = "https://127.0.0.1:$MOQ_HOST_PORT/",
+                // `localhost`, not `127.0.0.1` — see the doStart() return for
+                // why an IP-literal host breaks the TLS SNI / cert selection.
+                moqEndpoint = "https://localhost:$MOQ_HOST_PORT/",
            )
        }

@@ -107,16 +107,21 @@ fun buildQuicClientHello(
        ),
 ): TlsClientHello {
    val exts =
-        listOf(
-            TlsExtension(TlsConstants.EXT_SERVER_NAME, encodeServerNameExtension(serverName)),
-            TlsExtension(TlsConstants.EXT_SUPPORTED_VERSIONS, encodeSupportedVersionsExtensionClient()),
-            TlsExtension(TlsConstants.EXT_SUPPORTED_GROUPS, encodeSupportedGroupsX25519()),
-            TlsExtension(TlsConstants.EXT_SIGNATURE_ALGORITHMS, encodeSignatureAlgorithms()),
-            TlsExtension(TlsConstants.EXT_KEY_SHARE, encodeKeyShareClientX25519(x25519PublicKey)),
-            TlsExtension(TlsConstants.EXT_PSK_KEY_EXCHANGE_MODES, encodePskKeyExchangeModesDhe()),
-            TlsExtension(TlsConstants.EXT_ALPN, encodeAlpn(alpns)),
-            TlsExtension(TlsConstants.EXT_QUIC_TRANSPORT_PARAMETERS, quicTransportParams),
-        )
+        buildList {
+            // RFC 6066 §3: omit the SNI extension entirely for IP-literal
+            // targets — a literal address is not a valid host_name and a
+            // strict server rejects the extension, then fails cert selection.
+            if (!isIpLiteralHostname(serverName)) {
+                add(TlsExtension(TlsConstants.EXT_SERVER_NAME, encodeServerNameExtension(serverName)))
+            }
+            add(TlsExtension(TlsConstants.EXT_SUPPORTED_VERSIONS, encodeSupportedVersionsExtensionClient()))
+            add(TlsExtension(TlsConstants.EXT_SUPPORTED_GROUPS, encodeSupportedGroupsX25519()))
+            add(TlsExtension(TlsConstants.EXT_SIGNATURE_ALGORITHMS, encodeSignatureAlgorithms()))
+            add(TlsExtension(TlsConstants.EXT_KEY_SHARE, encodeKeyShareClientX25519(x25519PublicKey)))
+            add(TlsExtension(TlsConstants.EXT_PSK_KEY_EXCHANGE_MODES, encodePskKeyExchangeModesDhe()))
+            add(TlsExtension(TlsConstants.EXT_ALPN, encodeAlpn(alpns)))
+            add(TlsExtension(TlsConstants.EXT_QUIC_TRANSPORT_PARAMETERS, quicTransportParams))
+        }
    return TlsClientHello(random = random, cipherSuites = cipherSuites, extensions = exts)
 }

@@ -162,7 +167,10 @@ fun buildResumptionClientHelloBytes(
 ): ByteArray {
    val exts =
        buildList {
+            // RFC 6066 §3: no SNI for IP-literal targets — see buildQuicClientHello.
+            if (!isIpLiteralHostname(serverName)) {
                add(TlsExtension(TlsConstants.EXT_SERVER_NAME, encodeServerNameExtension(serverName)))
+            }
            add(TlsExtension(TlsConstants.EXT_SUPPORTED_VERSIONS, encodeSupportedVersionsExtensionClient()))
            add(TlsExtension(TlsConstants.EXT_SUPPORTED_GROUPS, encodeSupportedGroupsX25519()))
            add(TlsExtension(TlsConstants.EXT_SIGNATURE_ALGORITHMS, encodeSignatureAlgorithms()))
@@ -85,6 +85,32 @@ fun encodeServerNameExtension(hostName: String): ByteArray {
    return w.toByteArray()
 }

+/**
+ * RFC 6066 §3: the SNI `host_name` field MUST carry a DNS hostname — literal
+ * IPv4 and IPv6 addresses are explicitly NOT permitted. Strict servers reject
+ * an IP-literal SNI outright (e.g. `rustls`: "Illegal SNI extension: ignoring
+ * IP address presented as hostname"), which leaves the server with no SNI to
+ * key certificate selection on and stalls the handshake. Callers that build a
+ * ClientHello for an IP-literal target must therefore omit the `server_name`
+ * extension entirely; this predicate is the gate for that.
+ *
+ * Returns true for dotted-quad IPv4 (`127.0.0.1`) and for anything containing
+ * a `:` or wrapped in `[...]` (every IPv6 literal; no DNS hostname can contain
+ * a colon). Anything else — including a bare `localhost` — is treated as a
+ * hostname and gets a normal SNI.
+ */
+fun isIpLiteralHostname(host: String): Boolean {
+    if (host.isEmpty()) return false
+    // IPv6 literals always contain ':' (optionally bracketed); DNS names never do.
+    if (host.startsWith('[') || host.contains(':')) return true
+    // IPv4 dotted-quad: exactly four decimal octets in 0..255.
+    val parts = host.split('.')
+    if (parts.size != 4) return false
+    return parts.all { p ->
+        p.isNotEmpty() && p.length <= 3 && p.all { it in '0'..'9' } && p.toInt() in 0..255
+    }
+}
+
 /** Build the `supported_versions` extension carrying just TLS 1.3. */
 fun encodeSupportedVersionsExtensionClient(): ByteArray {
    val w = QuicWriter()