From c17fb66a4ea23026b2dd4404c9eda29255a6cd40 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 18:47:00 +0300 Subject: [PATCH 01/13] fix: recognize all UTS-46 dot separators in per-label IDNA leniency asciiLenientDomainToAscii split the raw domain on a literal U+002E before deciding, per label, whether to skip validation for an all-ASCII label. That missed the other three UTS-46 dot-separator variants (U+3002, U+FF0E, U+FF61), so a label boundary formed by one of them was invisible to the gate and the whole span got treated as one non-ASCII label, running the strict pipeline instead of getting the per-label leniency. Mirror domainToAscii's own step order instead: map the domain first, then split the mapped/NFC text into labels. Mapping ASCII input up front is safe since none of it changes under UseSTD3ASCIIRules=false beyond the case-folding asciiLowercase already does. --- .../kotlin/org/dexpace/kuri/idna/Idna.kt | 36 ++++++++++++------ .../kotlin/org/dexpace/kuri/idna/IdnTest.kt | 37 +++++++++++++++++++ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt index 92965d1..204be02 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt @@ -91,29 +91,43 @@ internal object Idna { } /** - * Splits [domain] on [LABEL_SEPARATOR] and resolves each label independently: an all-ASCII label - * is lowercased and kept unconditionally (never validated, matching the existing intentional - * leniency — see [domainToAsciiForUrl]); a label carrying any non-ASCII code point runs the full - * [domainToAscii] pipeline, and that label's failure fails the whole domain. Splitting first and - * running NFC per label (inside [domainToAscii]) rather than over the whole domain is equivalent - * here: `U+002E` never participates in a cross-label canonical composition. + * Maps [domain] first, then splits the mapped-and-NFC-normalized text on [LABEL_SEPARATOR] and + * resolves each label independently: an all-ASCII label is lowercased and kept unconditionally + * (never validated, matching the existing intentional leniency — see [domainToAsciiForUrl]); a + * label carrying any non-ASCII code point runs the full [domainToAscii] pipeline, and that + * label's failure fails the whole domain. Mapping before splitting (mirroring [domainToAscii]'s + * own step order) is required so a label boundary formed by one of the three non-ASCII UTS-46 + * dot-separator variants (U+3002, U+FF0E, U+FF61 — all [IdnaMapping.Mapped] to [LABEL_SEPARATOR]) + * is visible here too, not only inside the nested [domainToAscii] call on a non-ASCII label. + * Mapping the whole domain up front is safe: every ASCII code point reaching this function + * already passed the WHATWG forbidden-host-code-point filter upstream, and no ASCII code point + * maps outside ASCII or is dropped under `UseSTD3ASCIIRules = false`, so an all-ASCII label's + * content is unchanged in substance (only case-folded, which [asciiLowercase] already redoes + * harmlessly) and a `mapAll` failure can only occur where the nested [domainToAscii] on that same + * span would already have failed today. * - * @return the joined ASCII domain, or `null` on the first non-ASCII label's UTS-46 failure. + * @return the joined ASCII domain, or `null` on a mapping failure or the first non-ASCII label's + * UTS-46 failure. */ private fun asciiLenientDomainToAscii(domain: String): String? { - val labels = splitLabels(domain) + val mapped = mapAll(domain) ?: return null + val labels = splitLabels(Normalizer.nfc(mapped)) val out = ArrayList(labels.size) - for (label in labels) { + var index = 0 + var failed = false + while (index < labels.size && !failed) { + val label = labels[index] if (label.isAllAscii()) { out.add(asciiLowercase(label)) } else { when (val result = domainToAscii(label)) { is ParseResult.Ok -> out.add(result.value) - is ParseResult.Err -> return null + is ParseResult.Err -> failed = true } } + index++ } - return out.joinToString(LABEL_SEPARATOR) + return if (failed) null else out.joinToString(LABEL_SEPARATOR) } /** diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt index 0894067..db67239 100644 --- a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt @@ -9,6 +9,7 @@ import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertNotNull import kotlin.test.assertNull +import kotlin.test.assertTrue /** * Behavioural tests for the public [Idn] facade over the internal [Idna] engine (UTS-46, SPEC §7.4). @@ -45,6 +46,42 @@ class IdnTest { assertEquals("xn--a.xn--bcher-kva", Idn.toAscii(input).getOrNull()) } + @Test + fun `toAscii recognizes the ideographic full stop as a per-label leniency separator`() { + // U+3002 IDEOGRAPHIC FULL STOP is one of the three non-ASCII UTS-46 dot-separator + // variants mapped to U+002E; the per-label ASCII/non-ASCII gate must see the boundary it + // forms, not just a literal '.', so "xn--a" still gets the malformed-Punycode leniency. + val ideographicFullStop = Char(0x3002).toString() + val input = "xn--a" + ideographicFullStop + "b" + uUmlaut + "cher" + val expected = Idn.toAscii("xn--a.b" + uUmlaut + "cher") + + assertTrue(expected.isOk()) + assertEquals(expected.getOrNull(), Idn.toAscii(input).getOrNull()) + } + + @Test + fun `toAscii recognizes the fullwidth full stop as a per-label leniency separator`() { + // U+FF0E FULLWIDTH FULL STOP is another UTS-46 dot-separator variant mapped to U+002E. + val fullwidthFullStop = Char(0xFF0E).toString() + val input = "xn--a" + fullwidthFullStop + "b" + uUmlaut + "cher" + val expected = Idn.toAscii("xn--a.b" + uUmlaut + "cher") + + assertTrue(expected.isOk()) + assertEquals(expected.getOrNull(), Idn.toAscii(input).getOrNull()) + } + + @Test + fun `toAscii recognizes the halfwidth ideographic full stop as a per-label leniency separator`() { + // U+FF61 HALFWIDTH IDEOGRAPHIC FULL STOP is the third UTS-46 dot-separator variant mapped + // to U+002E. + val halfwidthIdeographicFullStop = Char(0xFF61).toString() + val input = "xn--a" + halfwidthIdeographicFullStop + "b" + uUmlaut + "cher" + val expected = Idn.toAscii("xn--a.b" + uUmlaut + "cher") + + assertTrue(expected.isOk()) + assertEquals(expected.getOrNull(), Idn.toAscii(input).getOrNull()) + } + @Test fun `toAscii fails on a disallowed code point`() { // U+0080 is a plain disallowed code point, so ToASCII must reject it fatally. From becf18cfcfe9497762e063bbc10b024a2a553aa3 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:05:41 +0300 Subject: [PATCH 02/13] feat: enforce RFC 3987 bidi and repertoire rules in Iri.toUri MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Iri.toUri previously percent-encoded any non-ASCII code point in an IRI's userinfo, path, query, or fragment without checking whether it was actually legal there. RFC 3987 restricts those components to the ucschar/iprivate repertoire (§2.2) and forbids bidirectional formatting characters like LRM and RLE anywhere in the IRI (§4.1). Add an IriRepertoire lookup for the fixed ucschar/iprivate ranges and the seven forbidden bidi characters, and run both checks in IriMapping before any host mapping or percent-encoding, so a violation is now a ParseResult.Err (UriParseError.IriInvalidCodePoint or IriBidiFormattingCharacter) instead of being silently encoded. The host stays out of this check since IDNA/UTS-46 already validates it independently. Threading component offsets through the IRI splitter so these errors can point at the exact offending index. The RFC's §4.2 per-component directionality rule is a SHOULD, not a MUST, and isn't enforced here; docs are updated to say so precisely. --- README.md | 4 +- kuri/api/jvm/kuri.api | 28 +++ kuri/api/kuri.klib.api | 32 ++++ .../commonMain/kotlin/org/dexpace/kuri/Iri.kt | 17 +- .../org/dexpace/kuri/error/UriParseError.kt | 32 +++- .../org/dexpace/kuri/parser/IriMapping.kt | 171 ++++++++++++++---- .../org/dexpace/kuri/parser/IriRepertoire.kt | 88 +++++++++ .../kotlin/org/dexpace/kuri/IriTest.kt | 60 +++++- 8 files changed, 383 insertions(+), 49 deletions(-) create mode 100644 kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriRepertoire.kt diff --git a/README.md b/README.md index 23d3e41..62ab693 100644 --- a/README.md +++ b/README.md @@ -473,7 +473,9 @@ kuri implements the standards below; per-standard conformance is measured in [Co | [RFC 3987][rfc3987] | IRIs — one-way `Iri` mapping, not a validating parser* | Supported | Default | | [WHATWG URL Standard][whatwg-url] | the `Url` model — parser, special schemes, canonical serialization | Conformant | Default | -\* kuri maps IRIs to URIs one-way (RFC 3987 §3.1/§3.2); it does not implement §4 validation (bidi, repertoire checks). +\* kuri maps IRIs to URIs one-way (RFC 3987 §3.1/§3.2), rejecting a §2.2 `ucschar`/`iprivate` +repertoire violation and a §4.1 bidi formatting character; it does not enforce §4.2's per-component +directionality restriction, which the RFC itself states as a SHOULD, not a MUST. **Hosts, internationalization, and IP addresses** diff --git a/kuri/api/jvm/kuri.api b/kuri/api/jvm/kuri.api index 2f55010..134cccd 100644 --- a/kuri/api/jvm/kuri.api +++ b/kuri/api/jvm/kuri.api @@ -373,6 +373,34 @@ public final class org/dexpace/kuri/error/UriParseError$InvalidScheme : org/dexp public fun toString ()Ljava/lang/String; } +public final class org/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter : org/dexpace/kuri/error/UriParseError { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter; + public static synthetic fun copy$default (Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter;IIILjava/lang/Object;)Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter; + public fun equals (Ljava/lang/Object;)Z + public final fun getAt ()I + public final fun getCodePoint ()I + public fun getMessage ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class org/dexpace/kuri/error/UriParseError$IriInvalidCodePoint : org/dexpace/kuri/error/UriParseError { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint; + public static synthetic fun copy$default (Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint;IIILjava/lang/Object;)Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint; + public fun equals (Ljava/lang/Object;)Z + public final fun getAt ()I + public final fun getCodePoint ()I + public fun getMessage ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class org/dexpace/kuri/error/UriParseError$MissingScheme : org/dexpace/kuri/error/UriParseError { public static final field INSTANCE Lorg/dexpace/kuri/error/UriParseError$MissingScheme; public fun equals (Ljava/lang/Object;)Z diff --git a/kuri/api/kuri.klib.api b/kuri/api/kuri.klib.api index d264fa1..c0a6ac7 100644 --- a/kuri/api/kuri.klib.api +++ b/kuri/api/kuri.klib.api @@ -160,6 +160,38 @@ sealed interface org.dexpace.kuri.error/UriParseError { // org.dexpace.kuri.erro final fun toString(): kotlin/String // org.dexpace.kuri.error/UriParseError.InvalidScheme.toString|toString(){}[0] } + final class IriBidiFormattingCharacter : org.dexpace.kuri.error/UriParseError { // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter|null[0] + constructor (kotlin/Int, kotlin/Int) // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.|(kotlin.Int;kotlin.Int){}[0] + + final val at // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.at|{}at[0] + final fun (): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.at.|(){}[0] + final val codePoint // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.codePoint|{}codePoint[0] + final fun (): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.codePoint.|(){}[0] + + final fun component1(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.component1|component1(){}[0] + final fun component2(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.component2|component2(){}[0] + final fun copy(kotlin/Int = ..., kotlin/Int = ...): org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.copy|copy(kotlin.Int;kotlin.Int){}[0] + final fun equals(kotlin/Any?): kotlin/Boolean // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.equals|equals(kotlin.Any?){}[0] + final fun hashCode(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.hashCode|hashCode(){}[0] + final fun toString(): kotlin/String // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.toString|toString(){}[0] + } + + final class IriInvalidCodePoint : org.dexpace.kuri.error/UriParseError { // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint|null[0] + constructor (kotlin/Int, kotlin/Int) // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.|(kotlin.Int;kotlin.Int){}[0] + + final val at // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.at|{}at[0] + final fun (): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.at.|(){}[0] + final val codePoint // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.codePoint|{}codePoint[0] + final fun (): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.codePoint.|(){}[0] + + final fun component1(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.component1|component1(){}[0] + final fun component2(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.component2|component2(){}[0] + final fun copy(kotlin/Int = ..., kotlin/Int = ...): org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.copy|copy(kotlin.Int;kotlin.Int){}[0] + final fun equals(kotlin/Any?): kotlin/Boolean // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.equals|equals(kotlin.Any?){}[0] + final fun hashCode(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.hashCode|hashCode(){}[0] + final fun toString(): kotlin/String // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.toString|toString(){}[0] + } + final object EmptyHost : org.dexpace.kuri.error/UriParseError { // org.dexpace.kuri.error/UriParseError.EmptyHost|null[0] final fun equals(kotlin/Any?): kotlin/Boolean // org.dexpace.kuri.error/UriParseError.EmptyHost.equals|equals(kotlin.Any?){}[0] final fun hashCode(): kotlin/Int // org.dexpace.kuri.error/UriParseError.EmptyHost.hashCode|hashCode(){}[0] diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/Iri.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/Iri.kt index 173eb64..024f28c 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/Iri.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/Iri.kt @@ -20,10 +20,13 @@ import kotlin.jvm.JvmStatic * "no RFC 3986 deviations" invariant (Appendix B): `Uri.parse` still *rejects* raw non-ASCII, and * only this facility accepts it. * - * The mapping is a faithful, uniform §3.1 transform, not an IRI *validator*: the host is processed + * The mapping is a §3.1 transform layered on two RFC 3987 validation passes: the host is processed * by IDNA/UTS-46 (via the engine's ToASCII), and every other component has its non-ASCII code points - * percent-encoded as UTF-8 octets. No `ucschar`/`iprivate` repertoire is consulted, so a non-ASCII - * code point outside the IRI character set is encoded rather than rejected. + * percent-encoded as UTF-8 octets — but only once that code point has cleared §2.2's `ucschar`/ + * `iprivate` repertoire (`iprivate` legal only in the query) and §4.1's ban on the seven bidi + * formatting characters (LRM, RLM, LRE, RLE, PDF, LRO, RLO) anywhere in the IRI. §4.2's per-component + * directionality restriction is intentionally not enforced: the RFC states it as a SHOULD, not a + * MUST, and it is not relevant to every use of an IRI (e.g. one never presented visually). * * @see Uri */ @@ -37,9 +40,11 @@ public object Iri { * (never double-encoded). The fully-ASCII result is then validated and stored by the strict [Uri] * engine, so the returned [Uri] holds only RFC 3986-valid text. * - * The transform can fail: an IDNA-invalid non-ASCII host is a [ParseResult.Err], as is a mapping - * whose expanded ASCII form is rejected by the strict engine — note that percent-encoding grows - * the input, so a very long all-non-ASCII IRI may exceed the engine's maximum input length. + * The transform can fail: an IDNA-invalid non-ASCII host is a [ParseResult.Err], as is a non-ASCII + * code point outside its component's §2.2 repertoire, a §4.1 bidi formatting character anywhere + * in [iri], or a mapping whose expanded ASCII form is rejected by the strict engine — note that + * percent-encoding grows the input, so a very long all-non-ASCII IRI may exceed the engine's + * maximum input length. * * @param iri the internationalized reference to convert; non-ASCII input is accepted. * @return [ParseResult.Ok] with the mapped [Uri], or [ParseResult.Err] when the IRI cannot be diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/error/UriParseError.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/error/UriParseError.kt index 5550048..91ca69a 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/error/UriParseError.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/error/UriParseError.kt @@ -16,7 +16,8 @@ package org.dexpace.kuri.error * The parser produces a fixed slice of this catalog: structural failures * ([InvalidScheme], [MissingScheme], [InvalidPercentEncoding], [InvalidPort]), * host failures ([EmptyHost], [InvalidHost] carrying a [HostError], and - * [ForbiddenHostCodePoint]), and the size failure [InputTooLong]. The hierarchy + * [ForbiddenHostCodePoint]), the size failure [InputTooLong], and the RFC 3987 IRI-conversion + * failures ([IriInvalidCodePoint], [IriBidiFormattingCharacter]). The hierarchy * is `sealed` so a `when` over it is exhaustive without an `else`; * adding a variant is an intentional, API-visible change. */ @@ -39,6 +40,8 @@ public sealed interface UriParseError { is InvalidHost -> "invalid host \"$host\": $reason" is ForbiddenHostCodePoint -> "forbidden host code point $codePoint at offset $at" is InputTooLong -> "input length $length exceeds maximum $max" + is IriInvalidCodePoint -> "code point $codePoint at offset $at is outside the iri repertoire" + is IriBidiFormattingCharacter -> "forbidden bidi formatting character $codePoint at offset $at" } /** @@ -132,4 +135,31 @@ public sealed interface UriParseError { public val length: Int, public val max: Int, ) : UriParseError + + /** + * A non-ASCII code point in an IRI component falls outside the RFC 3987 §2.2 repertoire that + * component permits: outside `ucschar` for userinfo, path, and fragment; outside `ucschar` and + * `iprivate` for the query (the only component `iprivate` is legal in). The host is excluded from + * this check — it is independently validated by the IDNA/UTS-46 pipeline. + * + * @property codePoint the offending Unicode scalar value. + * @property at the offset of the offending code unit in the original input. + */ + public data class IriInvalidCodePoint( + public val codePoint: Int, + public val at: Int, + ) : UriParseError + + /** + * An IRI contains one of the seven bidirectional formatting characters (LRM, RLM, LRE, RLE, PDF, + * LRO, RLO) that RFC 3987 §4.1 forbids anywhere in an IRI's logical (stored/transmitted) form — + * they affect visual rendering only and are never themselves part of the IRI. + * + * @property codePoint the offending bidi formatting code point. + * @property at the offset of the offending code unit in the original input. + */ + public data class IriBidiFormattingCharacter( + public val codePoint: Int, + public val at: Int, + ) : UriParseError } diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriMapping.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriMapping.kt index a85614a..dab9f85 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriMapping.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriMapping.kt @@ -6,12 +6,16 @@ package org.dexpace.kuri.parser import org.dexpace.kuri.Uri import org.dexpace.kuri.error.ParseResult +import org.dexpace.kuri.error.UriParseError import org.dexpace.kuri.error.map import org.dexpace.kuri.idna.Idna import org.dexpace.kuri.percent.PercentCodec import org.dexpace.kuri.percent.PercentEncodeSets import org.dexpace.kuri.scheme.Scheme import org.dexpace.kuri.scheme.schemeColonIndex +import org.dexpace.kuri.text.NON_ASCII_MIN +import org.dexpace.kuri.text.charCount +import org.dexpace.kuri.text.codePointAt import org.dexpace.kuri.text.isAllAscii /** The authority-introducing `//` and the number of code units it spans (RFC 3986 §3.2). */ @@ -33,6 +37,13 @@ private const val DOUBLE_SLASH_LENGTH: Int = 2 * * Percent-encoding grows the input, so the engine's `MAX_INPUT_LENGTH` bound applies to the * *expanded* ASCII string; a very long all-non-ASCII IRI may exceed it after encoding and be rejected. + * + * Before any mapping happens, [precheckError] runs two RFC 3987 checks over the raw text: + * [findBidiFormattingCharacter] rejects a bidi formatting character (LRM, RLM, LRE, RLE, PDF, LRO, + * RLO) anywhere in [toUri]'s input (§4.1's MUST-NOT), and [repertoireError] rejects a non-ASCII code + * point outside `ucschar`/`iprivate` in userinfo, path, query, or fragment (§2.2's grammar) — + * `iprivate` is accepted only in the query, per the ABNF. The host is exempt from the repertoire + * check: it is independently validated by IDNA/UTS-46. */ @Suppress("TooManyFunctions") // One cohesive coarse splitter decomposed into single-purpose helpers, // each well under the 60-line cap; mirrors UriParser's structural decomposition. @@ -41,22 +52,92 @@ internal object IriMapping { * Maps [iri] to a [Uri] under RFC 3987 §3.1, or returns the first fatal failure. * * @param iri the internationalized reference to convert; non-ASCII input is accepted. - * @return [ParseResult.Ok] with the mapped [Uri], or [ParseResult.Err] on an IDNA host failure - * or a strict-engine rejection of the expanded ASCII form. + * @return [ParseResult.Ok] with the mapped [Uri], or [ParseResult.Err] on a §4.1 bidi-formatting + * violation, a §2.2 repertoire violation, an IDNA host failure, or a strict-engine rejection of + * the expanded ASCII form. */ internal fun toUri(iri: String): ParseResult { val parts = decompose(iri) check(parts.host == null || parts.hasAuthority) { "a host requires an authority" } - val mappedHost = - when (parts.host) { - null -> null - else -> - when (val host = mapHost(parts.host)) { - is ParseResult.Err -> return host - is ParseResult.Ok -> host.value - } + val hostResult = resolveHost(precheckError(iri, parts), parts) + return when (hostResult) { + is ParseResult.Err -> hostResult + is ParseResult.Ok -> UriParser.parse(reassemble(parts, hostResult.value)).map { Uri(it) } + } + } + + /** + * The first §4.1 bidi-formatting or §2.2 repertoire violation in [iri], or `null` when both checks + * pass. Runs before any mapping so a rejected IRI never reaches IDNA or percent-encoding. + */ + private fun precheckError( + iri: String, + parts: IriParts, + ): UriParseError? = findBidiFormattingCharacter(iri) ?: repertoireError(parts) + + /** + * Resolves [parts]'s host to its mapped ASCII form, short-circuiting to [precheck] first: a + * precheck failure (bidi or repertoire) must win over a host failure so the earliest structural + * problem in [iri]'s own text is what gets reported, not one downstream of it. `null` means "no + * host" (a hostless reference), distinct from a mapping failure. + */ + private fun resolveHost( + precheck: UriParseError?, + parts: IriParts, + ): ParseResult = + when { + precheck != null -> ParseResult.Err(precheck) + parts.host == null -> ParseResult.Ok(null) + else -> mapHost(parts.host) + } + + /** + * The first of RFC 3987 §4.1's seven forbidden bidi formatting characters in [iri], or `null`. + * The rule is IRI-wide, not scoped to a single component, so this scans the raw, undecomposed text. + */ + private fun findBidiFormattingCharacter(iri: String): UriParseError? { + for (index in iri.indices) { + val codePoint = iri[index].code + if (IriRepertoire.isBidiFormattingCharacter(codePoint)) { + return UriParseError.IriBidiFormattingCharacter(codePoint, index) } - return UriParser.parse(reassemble(parts, mappedHost)).map { Uri(it) } + } + return null + } + + /** + * The first raw component (userinfo, path, query, then fragment) that carries a non-ASCII code + * point outside its RFC 3987 §2.2 repertoire, or `null`. The host is exempt: it is validated + * separately by the IDNA/UTS-46 pipeline in [mapHost]. + */ + private fun repertoireError(parts: IriParts): UriParseError? = + parts.userInfo?.let { validateRepertoire(it, allowIprivate = false) } + ?: validateRepertoire(parts.path, allowIprivate = false) + ?: parts.query?.let { validateRepertoire(it, allowIprivate = true) } + ?: parts.fragment?.let { validateRepertoire(it, allowIprivate = false) } + + /** + * Scans [located]'s text code point by code point, failing on the first non-ASCII value outside + * `ucschar` (and, when [allowIprivate], also outside `iprivate`) — see RFC 3987 §2.2. [Located.offset] + * is the component's start in the original IRI, so the reported failure locates the exact input index. + */ + private fun validateRepertoire( + located: Located, + allowIprivate: Boolean, + ): UriParseError? { + var index = 0 + while (index < located.text.length) { + val codePoint = codePointAt(located.text, index) + val legal = + codePoint < NON_ASCII_MIN || + IriRepertoire.isUcschar(codePoint) || + (allowIprivate && IriRepertoire.isIprivate(codePoint)) + if (!legal) { + return UriParseError.IriInvalidCodePoint(codePoint, located.offset + index) + } + index += charCount(codePoint) + } + return null } /** @@ -80,9 +161,9 @@ internal object IriMapping { val out = StringBuilder() parts.scheme?.let { out.append(it).append(':') } if (parts.hasAuthority) appendAuthority(out, parts, mappedHost) - out.append(encodeComponent(parts.path)) - parts.query?.let { out.append('?').append(encodeComponent(it)) } - parts.fragment?.let { out.append('#').append(encodeComponent(it)) } + out.append(encodeComponent(parts.path.text)) + parts.query?.let { out.append('?').append(encodeComponent(it.text)) } + parts.fragment?.let { out.append('#').append(encodeComponent(it.text)) } return out.toString() } @@ -95,7 +176,7 @@ internal object IriMapping { require(parts.hasAuthority) { "authority reassembly requires an authority" } val host = requireNotNull(mappedHost) { "a present authority requires a mapped host" } out.append(DOUBLE_SLASH) - parts.userInfo?.let { out.append(encodeComponent(it)).append('@') } + parts.userInfo?.let { out.append(encodeComponent(it.text)).append('@') } out.append(host) parts.port?.let { out.append(':').append(it) } } @@ -110,15 +191,16 @@ internal object IriMapping { /** Splits [iri] at its ASCII structural delimiters into the components the transform maps. */ private fun decompose(iri: String): IriParts { val hash = iri.indexOf('#') - val fragment = if (hash < 0) null else iri.substring(hash + 1) + val fragment = if (hash < 0) null else Located(iri.substring(hash + 1), hash + 1) val body = if (hash < 0) iri else iri.substring(0, hash) val mark = body.indexOf('?') - val query = if (mark < 0) null else body.substring(mark + 1) + val query = if (mark < 0) null else Located(body.substring(mark + 1), mark + 1) val hier = if (mark < 0) body else body.substring(0, mark) val scheme = detectScheme(hier) check(scheme == null || hier.length > scheme.length) { "a scheme requires a trailing colon" } + val restOffset = if (scheme == null) 0 else scheme.length + 1 val rest = if (scheme == null) hier else hier.substring(scheme.length + 1) - return splitAuthorityPath(scheme, rest, query, fragment) + return splitAuthorityPath(scheme, Located(rest, restOffset), query, fragment) } /** A scheme is the prefix before a `:` that precedes any `/`, and only when it is a valid scheme. */ @@ -132,17 +214,23 @@ internal object IriMapping { /** Routes [rest] to an authority split when it opens with `//`, else treats all of it as the path. */ private fun splitAuthorityPath( scheme: String?, - rest: String, - query: String?, - fragment: String?, + rest: Located, + query: Located?, + fragment: Located?, ): IriParts { - if (!rest.startsWith(DOUBLE_SLASH)) { + if (!rest.text.startsWith(DOUBLE_SLASH)) { return IriParts(scheme, hasAuthority = false, path = rest, query = query, fragment = fragment) } - val slash = rest.indexOf('/', DOUBLE_SLASH_LENGTH) - val end = if (slash < 0) rest.length else slash - val authority = splitAuthority(rest.substring(DOUBLE_SLASH_LENGTH, end)) - val path = if (slash < 0) "" else rest.substring(slash) + val slash = rest.text.indexOf('/', DOUBLE_SLASH_LENGTH) + val end = if (slash < 0) rest.text.length else slash + val authorityText = rest.text.substring(DOUBLE_SLASH_LENGTH, end) + val authority = splitAuthority(Located(authorityText, rest.offset + DOUBLE_SLASH_LENGTH)) + val path = + if (slash < 0) { + Located("", rest.offset + rest.text.length) + } else { + Located(rest.text.substring(slash), rest.offset + slash) + } return IriParts( scheme = scheme, hasAuthority = true, @@ -156,16 +244,16 @@ internal object IriMapping { } /** Splits authority text into userinfo (before the LAST `@`) and host[:port]. */ - private fun splitAuthority(authority: String): Authority { - val at = authority.lastIndexOf('@') - val userInfo = if (at < 0) null else authority.substring(0, at) - val hostPort = if (at < 0) authority else authority.substring(at + 1) + private fun splitAuthority(authority: Located): Authority { + val at = authority.text.lastIndexOf('@') + val userInfo = if (at < 0) null else Located(authority.text.substring(0, at), authority.offset) + val hostPort = if (at < 0) authority.text else authority.text.substring(at + 1) return if (hostPort.startsWith('[')) splitBracketed(userInfo, hostPort) else splitPlain(userInfo, hostPort) } /** A non-bracketed host ends at the first `:`, whose tail is the port. */ private fun splitPlain( - userInfo: String?, + userInfo: Located?, hostPort: String, ): Authority { val colon = hostPort.indexOf(':') @@ -177,7 +265,7 @@ internal object IriMapping { /** A bracketed literal's port colon, if any, immediately follows the closing `]`. */ private fun splitBracketed( - userInfo: String?, + userInfo: Located?, hostPort: String, ): Authority { val close = hostPort.indexOf(']') @@ -188,21 +276,30 @@ internal object IriMapping { return Authority(userInfo, host, port) } + /** + * A raw substring of the original IRI together with its start offset, so a later validation pass + * (the RFC 3987 §2.2 repertoire check) can report the exact input index of a rejected code point. + */ + private data class Located( + val text: String, + val offset: Int, + ) + /** The located structural components of an IRI, each carried raw (pre-transform). */ private data class IriParts( val scheme: String?, val hasAuthority: Boolean, - val userInfo: String? = null, + val userInfo: Located? = null, val host: String? = null, val port: String? = null, - val path: String, - val query: String?, - val fragment: String?, + val path: Located, + val query: Located? = null, + val fragment: Located? = null, ) /** The located authority sub-components: raw userinfo (or `null`), raw host, and raw port (or `null`). */ private data class Authority( - val userInfo: String?, + val userInfo: Located?, val host: String, val port: String?, ) diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriRepertoire.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriRepertoire.kt new file mode 100644 index 0000000..d8c63ba --- /dev/null +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriRepertoire.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2026 dexpace and Omar Aljarrah + * SPDX-License-Identifier: MIT + */ +package org.dexpace.kuri.parser + +/** Separates the inclusive `-` records within one of this file's hex range blobs. */ +private const val RECORD_SEPARATOR: Char = ';' + +/** Separates the start and end hex bounds of one inclusive range record. */ +private const val RANGE_SEPARATOR: Char = '-' + +/** Radix the hex bounds and bidi-formatting code points are parsed in. */ +private const val RADIX_HEX: Int = 16 + +/** + * The RFC 3987 §2.2 `ucschar`/`iprivate` repertoire and the §4.1 bidi-formatting-character set. + * + * Both tables are fixed by the RFC's own ABNF (not tied to any Unicode version), so unlike the + * generated IDNA/UTS-46 tables they never need regeneration. They are still stored as parsed hex + * text rather than `IntRange` literals, mirroring the codegen'd IDNA range tables: it keeps every + * bound traceable one-for-one to the ABNF quoted below without a wall of unnamed numeric literals. + */ +internal object IriRepertoire { + /** + * The `ucschar` ranges (RFC 3987 §2.2): + * `%xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD / + * %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD / %x70000-7FFFD / %x80000-8FFFD / + * %x90000-9FFFD / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD / %xD0000-DFFFD / + * %xE1000-EFFFD`. These are the code points [iunreserved] may hold beyond + * `ALPHA`/`DIGIT`/`"-"`/`"."`/`"_"`/`"~"`, legal in every non-host component this facility maps. + */ + private val ucscharRanges: List = + decodeRanges( + "A0-D7FF;F900-FDCF;FDF0-FFEF;10000-1FFFD;20000-2FFFD;30000-3FFFD;40000-4FFFD;" + + "50000-5FFFD;60000-6FFFD;70000-7FFFD;80000-8FFFD;90000-9FFFD;A0000-AFFFD;" + + "B0000-BFFFD;C0000-CFFFD;D0000-DFFFD;E1000-EFFFD", + ) + + /** + * The `iprivate` ranges (RFC 3987 §2.2): `%xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD`. Per + * the ABNF, `iprivate` appears only in `iquery` (`iquery = *( ipchar / iprivate / "/" / "?" )`) — + * it is not part of `ipchar`, so it is illegal in userinfo, path, and fragment even though it is + * legal in the query. + */ + private val iprivateRanges: List = decodeRanges("E000-F8FF;F0000-FFFFD;100000-10FFFD") + + /** + * The seven bidirectional formatting characters RFC 3987 §4.1 forbids anywhere in an IRI: LRM + * (U+200E), RLM (U+200F), LRE (U+202A), RLE (U+202B), PDF (U+202C), LRO (U+202D), RLO (U+202E). + * They affect visual rendering only and must never appear in the logical (stored/transmitted) form. + */ + private val bidiFormattingCharacters: Set = + "200E,200F,202A,202B,202C,202D,202E".split(',').map { it.toInt(RADIX_HEX) }.toSet() + + /** + * True when [codePoint] is in one of the `ucschar` ranges (RFC 3987 §2.2). + * + * @param codePoint the Unicode scalar value to test. + * @return `true` iff [codePoint] falls within a `ucschar` range. + */ + internal fun isUcschar(codePoint: Int): Boolean = ucscharRanges.any { codePoint in it } + + /** + * True when [codePoint] is in one of the `iprivate` ranges (RFC 3987 §2.2), legal only in the + * query component. + * + * @param codePoint the Unicode scalar value to test. + * @return `true` iff [codePoint] falls within an `iprivate` range. + */ + internal fun isIprivate(codePoint: Int): Boolean = iprivateRanges.any { codePoint in it } + + /** + * True when [codePoint] is one of the seven bidi formatting characters RFC 3987 §4.1 forbids. + * + * @param codePoint the Unicode scalar value to test. + * @return `true` iff [codePoint] is LRM, RLM, LRE, RLE, PDF, LRO, or RLO. + */ + internal fun isBidiFormattingCharacter(codePoint: Int): Boolean = codePoint in bidiFormattingCharacters + + /** Decodes a semicolon-joined `-` hex blob into its inclusive [IntRange]s. */ + private fun decodeRanges(blob: String): List = + blob.split(RECORD_SEPARATOR).map { record -> + val dash = record.indexOf(RANGE_SEPARATOR) + check(dash > 0) { "malformed range record: $record" } + record.substring(0, dash).toInt(RADIX_HEX)..record.substring(dash + 1).toInt(RADIX_HEX) + } +} diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt index 3347126..78fb7a9 100644 --- a/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt @@ -101,13 +101,65 @@ class IriTest { } @Test - fun `encodes any non-ascii code point even outside the iri repertoire`() { - // U+FDD0 is a noncharacter: neither ucschar nor iprivate, yet the converter still encodes it. + fun `rejects a non-ascii code point outside the iri repertoire`() { + // U+FDD0 is a noncharacter: neither ucschar nor iprivate, so RFC 3987 §2.2 excludes it. val nonCharacter = Char(0xFDD0).toString() + val iri = "http://h/$nonCharacter" - val uri = Iri.toUri("http://h/$nonCharacter").getOrThrow() + val error = assertIs(assertIs(Iri.toUri(iri)).error) - assertEquals("/%EF%B7%90", uri.encodedPath) + assertEquals(0xFDD0, error.codePoint) + assertEquals(iri.indexOf(nonCharacter), error.at) + } + + @Test + fun `accepts an iprivate code point only in the query component`() { + // U+E000 is iprivate: legal in iquery (RFC 3987 §2.2) but not in path, userinfo, or fragment. + val iprivate = Char(0xE000).toString() + + val uri = Iri.toUri("http://h/?q=$iprivate").getOrThrow() + + assertEquals("q=%EE%80%80", uri.query) + } + + @Test + fun `rejects an iprivate code point in the path`() { + val iprivate = Char(0xE000).toString() + val iri = "http://h/$iprivate" + + val error = assertIs(assertIs(Iri.toUri(iri)).error) + + assertEquals(0xE000, error.codePoint) + assertEquals(iri.indexOf(iprivate), error.at) + } + + @Test + fun `rejects an iprivate code point in userinfo`() { + val iprivate = Char(0xE000).toString() + val iri = "http://u$iprivate@h/" + + assertIs(assertIs(Iri.toUri(iri)).error) + } + + @Test + fun `rejects an iprivate code point in the fragment`() { + val iprivate = Char(0xE000).toString() + val iri = "http://h/#$iprivate" + + assertIs(assertIs(Iri.toUri(iri)).error) + } + + @Test + fun `rejects a bidi formatting character anywhere in the iri`() { + // U+200E LEFT-TO-RIGHT MARK is one of the seven formatting characters RFC 3987 §4.1 forbids. + val lrm = Char(0x200E).toString() + val iri = "http://h/$lrm" + + val error = + assertIs(assertIs(Iri.toUri(iri)).error) + + assertEquals(0x200E, error.codePoint) + assertEquals(iri.indexOf(lrm), error.at) } @Test From 5fead5794597a893f2a0dc3453d346d0c7a9a23d Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:17:14 +0300 Subject: [PATCH 03/13] docs: clarify idnaref is not an independent IDNA conformance oracle tools/internal/idnaref re-derives kuri's own domainToAscii algorithm as a second implementation, so a bug shared between the Kotlin engine and its Go port would pass the known-failures ratchet undetected. Its value is catching porting slips between the two, not verifying the algorithm against an outside source. WPT is the actual independent oracle IdnaConformanceTest checks kuri against; make that explicit in the package doc, the codegen generator, IdnaConformanceTest's class doc, and the Unicode-update runbook instead of leaving it implied. --- docs/idna-unicode-update.md | 4 +++- .../kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt | 9 +++++++++ tools/internal/codegen/conformance.go | 4 ++++ tools/internal/idnaref/reference.go | 9 +++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/idna-unicode-update.md b/docs/idna-unicode-update.md index 06e7a0e..ce0f7db 100644 --- a/docs/idna-unicode-update.md +++ b/docs/idna-unicode-update.md @@ -30,7 +30,9 @@ The generators are Go programs under `tools/` (module `github.com/dexpace/kuri/tools`), driven by `tools/internal/codegen`. That package reads raw UCD via `tools/internal/ucd` and derives the conformance baseline via `tools/internal/idnaref`, a faithful Go port of kuri's runtime IDNA -engine. +engine — a second implementation of kuri's own algorithm, not an independent +oracle. WPT (below) is the independent oracle; idnaref only narrows the +known-failures residual within whatever WPT doesn't already exercise. ## Source of truth diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt index 63902e4..97c5b47 100644 --- a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt @@ -31,6 +31,15 @@ import kotlin.test.assertTrue * by [Idna.domainToAscii] (so the corpus's required rejection is a host obligation, not a ToASCII one) * yet rejected by [UrlHostParser] under the `Url` profile. The suite also ratchets: it fails if any other * case regresses, or if the residual drifts from the live failing set. + * + * WPT is the independent oracle this suite checks kuri against, not `tools/internal/idnaref` (the Go + * reference the codegen generator uses to derive [IDNA_KNOWN_FAILURES]). `idnaref` is a faithful Go port + * of this same [Idna.domainToAscii] algorithm — a second implementation, not a second spec reading — so a + * conceptual bug shared between the Kotlin engine and its port would reproduce identically in both and + * pass the derivation undetected; its value is catching a porting slip between the two, not verifying the + * algorithm against an outside source. That verification is what running every case here directly against + * the WPT corpus itself provides. `IdnaIcu4jDifferentialTest` (JVM-only) adds a further, genuinely + * third-party cross-check against ICU4J's UTS-46 implementation over the same corpus. */ class IdnaConformanceTest { private val residual: Set = IDNA_KNOWN_FAILURES diff --git a/tools/internal/codegen/conformance.go b/tools/internal/codegen/conformance.go index fc9028d..b1a0807 100644 --- a/tools/internal/codegen/conformance.go +++ b/tools/internal/codegen/conformance.go @@ -15,6 +15,10 @@ // of kuri's Idna.domainToAscii is run over the corpus and the inputs it fails // become the tracked set. Because the reference matches kuri exactly, that set // equals kuri's live failing set, which IdnaConformanceTest asserts against. +// +// idnaref is a second implementation of kuri's own algorithm, not an independent +// oracle — see the idnaref package doc. WPT is the independent oracle IdnaConformanceTest +// checks kuri against; idnaref only narrows the residual within whatever WPT doesn't cover. package codegen diff --git a/tools/internal/idnaref/reference.go b/tools/internal/idnaref/reference.go index 517922d..76d9349 100644 --- a/tools/internal/idnaref/reference.go +++ b/tools/internal/idnaref/reference.go @@ -6,6 +6,15 @@ // tables decode. The codegen conformance generator runs it over the WPT corpora // to derive the tracked known-failures baseline. It depends only on internal/ucd // for its table data. +// +// idnaref is not an independent oracle: it re-derives kuri's own algorithm as a +// second implementation, so a conceptual bug shared between the Kotlin engine and +// this port (e.g. a shared misreading of the UTS-46 algorithm) would reproduce +// identically in both and pass the ratchet undetected. Its value is catching +// porting slips between the two — a divergence that is NOT shared — not verifying +// the algorithm itself. WPT is the real independent oracle here: IdnaConformanceTest +// checks every corpus case directly against it, and idnaref only narrows the +// residual within whatever WPT doesn't already exercise. package idnaref import ( From 6b94880d62f27a66fcaff08953d7cf9badee64f6 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:17:31 +0300 Subject: [PATCH 04/13] test: cross-check IDNA against ICU4J over the WPT corpus Add icu4j as a jvmTest-only dependency and run kuri's Idna.domainToAscii against ICU4J's UTS-46 implementation over every WPT IDNA case, as a genuinely third-party check alongside the existing WPT and idnaref comparisons. icu4j has no multiplatform artifact so it is scoped to the JVM test classpath only. kuri and ICU4J disagree in two structural, documented ways rather than everywhere: ICU4J's UTS46 has no switch to disable CheckHyphens or VerifyDnsLength, so it always rejects a leading/trailing hyphen, a "??--" label, an empty label, or a length overflow that kuri's Url profile deliberately permits; and ICU4J 77.1 bundles Unicode 16.0 data against kuri's 17.0, so a few inputs carrying newly assigned CJK ideographs disagree until ICU4J catches up. Both are asserted, not silently skipped: the first is filtered by ICU4J's own error codes with a one-line reason per code, and the second is pinned to an exact input list that ratchets if either side's Unicode version changes. --- gradle/libs.versions.toml | 5 + kuri/build.gradle.kts | 4 + .../kuri/idna/IdnaIcu4jDifferentialTest.kt | 111 ++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 kuri/src/jvmTest/kotlin/org/dexpace/kuri/idna/IdnaIcu4jDifferentialTest.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a1d6fe5..401f4c5 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,10 +7,15 @@ kover = "0.9.8" binary-compat = "0.18.1" dokka = "2.0.0" maven-publish = "0.30.0" +icu4j = "77.1" [libraries] # kotlin-test is pulled in via the kotlin("test") MPP helper; listed for reference. kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" } +# jvmTest-only: an independent third-party UTS-46 implementation to cross-check kuri's IDNA +# engine against (see IdnaIcu4jDifferentialTest). No multiplatform artifact exists, so this +# must never be referenced from commonMain/commonTest. +icu4j = { module = "com.ibm.icu:icu4j", version.ref = "icu4j" } [plugins] kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } diff --git a/kuri/build.gradle.kts b/kuri/build.gradle.kts index c0ec576..b7a3788 100644 --- a/kuri/build.gradle.kts +++ b/kuri/build.gradle.kts @@ -160,6 +160,10 @@ kotlin { // on the Java test's compile classpath, at the version the Kotlin release was built for. jvmTest.dependencies { implementation(kotlin("test-junit")) + // Independent third-party UTS-46 oracle for IdnaIcu4jDifferentialTest (issue #66): + // icu4j has no multiplatform artifact, so it is scoped to the JVM test classpath only + // and must never be referenced from commonMain/commonTest. + implementation(libs.icu4j) } // Instrumented tests run under AndroidJUnitRunner (JUnit4), so bind kotlin.test to JUnit4 // and pull in the AndroidX test runtime that provides the runner. diff --git a/kuri/src/jvmTest/kotlin/org/dexpace/kuri/idna/IdnaIcu4jDifferentialTest.kt b/kuri/src/jvmTest/kotlin/org/dexpace/kuri/idna/IdnaIcu4jDifferentialTest.kt new file mode 100644 index 0000000..fe968c7 --- /dev/null +++ b/kuri/src/jvmTest/kotlin/org/dexpace/kuri/idna/IdnaIcu4jDifferentialTest.kt @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2026 dexpace and Omar Aljarrah + * SPDX-License-Identifier: MIT + */ +package org.dexpace.kuri.idna + +import com.ibm.icu.text.IDNA +import org.dexpace.kuri.error.ParseResult +import kotlin.test.Test +import kotlin.test.assertTrue + +/** + * Cross-checks [Idna.domainToAscii] against ICU4J's UTS-46 implementation over the WPT IDNA corpus + * ([IDNA_CONFORMANCE_CASES]) — a genuinely third-party oracle, unlike `tools/internal/idnaref` (see + * [IdnaConformanceTest]), which only re-derives kuri's own algorithm as a second implementation. + * + * kuri and ICU4J legitimately disagree in two structural, documented ways rather than everywhere they + * differ being swallowed as noise: + * - ICU4J's UTS46 implementation has no switch to disable `CheckHyphens` or `VerifyDnsLength`: it always + * rejects a leading/trailing hyphen, a "??--" third/fourth-position hyphen, an empty label, or a + * length overflow ([structuralOptionErrors]). kuri's `Url` profile deliberately sets both to `false` + * (WHATWG requires the lenient defaults for URL hosts), so a case built to exercise one of those + * relaxations disagrees by construction, not by defect. + * - ICU4J 77.1 bundles Unicode 16.0 data; kuri bundles Unicode 17.0. A handful of corpus inputs carry + * CJK ideographs newly assigned in 17.0 that ICU4J's older tables still treat as unassigned + * ([unicodeVersionSkewInputs]), pinned by exact input and ratcheted so a Unicode-version change on + * either side forces a conscious update instead of a silent drift. + * + * Every other disagreement fails the suite: nothing outside those two documented causes is swallowed. + */ +class IdnaIcu4jDifferentialTest { + private val icu: IDNA = IDNA.getUTS46Instance(IDNA.DEFAULT or IDNA.CHECK_BIDI or IDNA.CHECK_CONTEXTJ) + + /** + * ICU4J [IDNA.Error] categories ICU4J's UTS46 implementation always enforces with no equivalent of + * kuri's `Url`-profile `CheckHyphens = false` / `VerifyDnsLength = false` relaxations, each mapped to + * the kuri profile rule that explains a case flagged with it. + */ + private val structuralOptionErrors: Map = + mapOf( + IDNA.Error.LEADING_HYPHEN to "CheckHyphens=false permits a leading hyphen; ICU4J has no such switch", + IDNA.Error.TRAILING_HYPHEN to "CheckHyphens=false permits a trailing hyphen; ICU4J has no such switch", + IDNA.Error.HYPHEN_3_4 to "CheckHyphens=false permits a 3rd/4th-position hyphen; ICU4J has no such switch", + IDNA.Error.EMPTY_LABEL to "VerifyDnsLength=false permits an empty label; ICU4J always rejects one", + IDNA.Error.LABEL_TOO_LONG to "VerifyDnsLength=false skips the 63-octet label cap ICU4J always enforces", + IDNA.Error.DOMAIN_NAME_TOO_LONG to "VerifyDnsLength=false skips the 255-octet domain cap ICU4J enforces", + ) + + /** + * Exact WPT inputs where kuri (Unicode 17.0) and ICU4J 77.1 (Unicode 16.0) disagree solely because + * ICU4J's bundled tables still treat a CJK ideograph the input carries (U+32931 or U+32B9A) as + * unassigned. Copied verbatim from [IDNA_CONFORMANCE_CASES] so a corpus regeneration cannot silently + * rename them out from under this set. + */ + private val unicodeVersionSkewInputs: Set = + setOf( + "𲤱20.音.ꡦ1.", // U+32931 label; kuri accepts, ICU4J: DISALLOWED (unassigned) + "xn--20-9802c.xn--0w5a.xn--1-eg4e.", // its ToASCII form, re-decoded and re-checked as an A-label + "xn--9-i0j5967eg3qz.ss", // ToASCII form of the U+32B9A case below, re-checked as an A-label + "𲮚9ꍩ៓.ss", // U+32B9A label; kuri accepts, ICU4J: DISALLOWED (unassigned) + "𲮚9ꍩ៓.SS", // same, uppercase SS label (kuri case-folds before comparing) + ) + + private fun icuToAscii(input: String): Pair> { + val info = IDNA.Info() + val out = StringBuilder() + icu.nameToASCII(input, out, info) + val value = if (info.hasErrors()) null else out.toString() + return value to info.errors + } + + private fun isStructuralOptionDifference(errors: Set): Boolean = + errors.isNotEmpty() && errors.all { it in structuralOptionErrors } + + private fun isExplainedDifference( + input: String, + icuErrors: Set, + ): Boolean = isStructuralOptionDifference(icuErrors) || input in unicodeVersionSkewInputs + + @Test + fun `kuri and ICU4J agree on every WPT case outside the documented differences`() { + val unexplained = + IDNA_CONFORMANCE_CASES.mapNotNull { case -> + val kuriValue = (Idna.domainToAscii(case.input) as? ParseResult.Ok)?.value + val (icuValue, icuErrors) = icuToAscii(case.input) + if (kuriValue == icuValue || isExplainedDifference(case.input, icuErrors)) { + null + } else { + "input=<${case.input}> kuri=<$kuriValue> icu=<$icuValue> icuErrors=$icuErrors" + } + } + assertTrue(unexplained.isEmpty(), "unexplained kuri/ICU4J disagreements:\n${unexplained.joinToString("\n")}") + } + + @Test + fun `every documented Unicode version skew input is a live kuri and ICU4J disagreement`() { + // Ratchet: a kuri or ICU4J Unicode-version bump that resolves the skew makes kuri and ICU4J agree + // again, and this then fails until the now-stale entry is removed from the documented set above. + unicodeVersionSkewInputs.forEach { input -> + val kuriValue = (Idna.domainToAscii(input) as? ParseResult.Ok)?.value + val (icuValue, _) = icuToAscii(input) + assertTrue(kuriValue != icuValue, "expected a live kuri/ICU4J disagreement for <$input>") + } + } + + @Test + fun `the corpus dwarfs the documented Unicode version skew`() { + assertTrue(IDNA_CONFORMANCE_CASES.size > 10 * unicodeVersionSkewInputs.size) + assertTrue(unicodeVersionSkewInputs.isNotEmpty()) + } +} From 5ae1cafe7d6265867d6feb08156d9bbb6b85fc93 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:28:20 +0300 Subject: [PATCH 05/13] ci: wire up android apiDump/apiCheck binary-compatibility-validator's built-in Kotlin Multiplatform support only recognizes an Android target compilation named "release" (the classic AGP com.android.library + androidTarget() build-variant naming). The AGP KMP library plugin used here (com.android.kotlin.multiplatform.library) is single-variant and names its production compilation "main" instead, so BCV never matched it and silently skipped the android surface: no androidApiBuild/androidApiCheck/androidApiDump tasks ever existed, which is how kuri/api/android/kuri.api went stale with nothing to regenerate or check it. Wire an equivalent android leg by hand with BCV's own public task classes (KotlinApiBuildTask/KotlinApiCompareTask) against the android target's compileAndroidMain output, using the same api/android/kuri.api layout BCV's other target legs already use, and hook it into the existing apiDump/apiCheck entry points so it runs as part of the same gate rather than needing a separate invocation. --- kuri/build.gradle.kts | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/kuri/build.gradle.kts b/kuri/build.gradle.kts index b7a3788..0bcc562 100644 --- a/kuri/build.gradle.kts +++ b/kuri/build.gradle.kts @@ -7,6 +7,8 @@ import com.vanniktech.maven.publish.JavadocJar import com.vanniktech.maven.publish.KotlinMultiplatform import com.vanniktech.maven.publish.SonatypeHost import kotlinx.kover.gradle.plugin.dsl.CoverageUnit +import kotlinx.validation.KotlinApiBuildTask +import kotlinx.validation.KotlinApiCompareTask import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.dsl.KotlinVersion @@ -246,6 +248,61 @@ apiValidation { } } +// binary-compatibility-validator's built-in multiplatform auto-configuration only recognizes an +// Android target compilation literally named "release" (the classic AGP `com.android.library` + +// `androidTarget()` build-variant naming). The AGP KMP library plugin used here +// (`com.android.kotlin.multiplatform.library`) is single-variant and names its production +// compilation "main" instead, so that auto-configuration never matches it, `androidApiBuild`/ +// `androidApiCheck`/`androidApiDump` tasks never get created, and `apiCheck`/`apiDump` silently +// skip the android surface entirely (see the upstream gap tracked as KT-71172; verified locally +// by inspecting the plugin's target-matching source — it hasn't changed as of BCV 0.18.1). That +// silent skip is exactly how `kuri/api/android/kuri.api` went stale: nothing ever regenerated or +// checked it. Wire an equivalent android leg by hand using BCV's own public task classes, +// `KotlinApiBuildTask`/`KotlinApiCompareTask`, mirroring the classpath and directory layout +// (`api/android/kuri.api`) its Kotlin-target legs already use, and hook the result into the same +// `apiDump`/`apiCheck` entry points so android gets no special-cased invocation. +val androidAbiRuntimeClasspath: NamedDomainObjectProvider = + configurations.register("androidAbiRuntimeClasspath") { + description = "Runtime classpath for the manually wired android apiBuild/apiCheck tasks." + isCanBeConsumed = false + isCanBeResolved = true + isVisible = false + } + +dependencies.apply { + add(androidAbiRuntimeClasspath.name, "org.ow2.asm:asm:9.6") + add(androidAbiRuntimeClasspath.name, "org.ow2.asm:asm-tree:9.6") + add(androidAbiRuntimeClasspath.name, "org.jetbrains.kotlin:kotlin-metadata-jvm:${libs.versions.kotlin.get()}") +} + +val androidApiBuild: TaskProvider = + tasks.register("androidApiBuild") { + group = "verification" + description = "Builds the Kotlin API dump for the android target's 'main' compilation of kuri." + inputClassesDirs.from(tasks.named("compileAndroidMain").map { it.outputs.files }) + outputApiFile.set(layout.buildDirectory.file("kotlin/abi-android/kuri.api")) + runtimeClasspath.from(androidAbiRuntimeClasspath) + } + +val androidApiCheck: TaskProvider = + tasks.register("androidApiCheck") { + group = "verification" + description = "Checks the android target's public API against api/android/kuri.api." + projectApiFile.set(layout.projectDirectory.file("api/android/kuri.api")) + generatedApiFile.set(androidApiBuild.flatMap { it.outputApiFile }) + } + +val androidApiDump: TaskProvider = + tasks.register("androidApiDump") { + group = "other" + description = "Copies the android target's generated API dump into api/android/kuri.api." + from(androidApiBuild.flatMap { it.outputApiFile }) + into(layout.projectDirectory.dir("api/android")) + } + +tasks.named("apiCheck") { dependsOn(androidApiCheck) } +tasks.named("apiDump") { dependsOn(androidApiDump) } + // Publish every Kotlin Multiplatform target (plus the root `kotlinMultiplatform` publication) to Maven // Central through the Central Portal. The vanniktech plugin wires the per-target publications, a // Dokka-generated Javadoc jar and GPG signing; coordinates derive from the module `group` and the From 37d0d8f62acd8337df03c30ce3f8ec74e3f34e52 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:28:26 +0300 Subject: [PATCH 06/13] chore: regenerate kuri/api/android/kuri.api Bring the android API snapshot current now that androidApiDump/apiCheck actually run it: it now matches api/jvm/kuri.api's public surface (minus the jvmMain-only java.net interop, which isn't compiled for android) and includes every API addition landed on this branch so far, including ValidationError.INVALID_CREDENTIALS. --- kuri/api/android/kuri.api | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/kuri/api/android/kuri.api b/kuri/api/android/kuri.api index fa8e1ff..87de2b6 100644 --- a/kuri/api/android/kuri.api +++ b/kuri/api/android/kuri.api @@ -362,6 +362,34 @@ public final class org/dexpace/kuri/error/UriParseError$InvalidScheme : org/dexp public fun toString ()Ljava/lang/String; } +public final class org/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter : org/dexpace/kuri/error/UriParseError { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter; + public static synthetic fun copy$default (Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter;IIILjava/lang/Object;)Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter; + public fun equals (Ljava/lang/Object;)Z + public final fun getAt ()I + public final fun getCodePoint ()I + public fun getMessage ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class org/dexpace/kuri/error/UriParseError$IriInvalidCodePoint : org/dexpace/kuri/error/UriParseError { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint; + public static synthetic fun copy$default (Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint;IIILjava/lang/Object;)Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint; + public fun equals (Ljava/lang/Object;)Z + public final fun getAt ()I + public final fun getCodePoint ()I + public fun getMessage ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class org/dexpace/kuri/error/UriParseError$MissingScheme : org/dexpace/kuri/error/UriParseError { public static final field INSTANCE Lorg/dexpace/kuri/error/UriParseError$MissingScheme; public fun equals (Ljava/lang/Object;)Z @@ -376,6 +404,7 @@ public final class org/dexpace/kuri/error/UriSyntaxException : java/lang/Illegal public final class org/dexpace/kuri/error/ValidationError : java/lang/Enum { public static final field BACKSLASH_AS_SOLIDUS Lorg/dexpace/kuri/error/ValidationError; + public static final field INVALID_CREDENTIALS Lorg/dexpace/kuri/error/ValidationError; public static final field INVALID_URL_UNIT Lorg/dexpace/kuri/error/ValidationError; public static final field LEADING_OR_TRAILING_C0_CONTROL_OR_SPACE Lorg/dexpace/kuri/error/ValidationError; public static final field MISSING_AUTHORITY_SLASHES Lorg/dexpace/kuri/error/ValidationError; @@ -519,6 +548,7 @@ public final class org/dexpace/kuri/query/QueryParameters : java/lang/Iterable, public static final fun parse (Ljava/lang/String;)Lorg/dexpace/kuri/query/QueryParameters; public static final fun parseForm (Ljava/lang/String;)Lorg/dexpace/kuri/query/QueryParameters; public final fun size ()I + public final fun split (Ljava/lang/String;C)Ljava/util/List; public final fun toFormUrlEncoded ()Ljava/lang/String; public final fun toMap ()Ljava/util/Map; public final fun toQueryString ()Ljava/lang/String; From 710064fe734f7af8e753b46e0eedba437cf2c5cb Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:36:44 +0300 Subject: [PATCH 07/13] chore: vendor the WPT urlencoded-parser corpus and add its generator x-www-form-urlencoded parsing had no conformance corpus wired up, unlike the URL/IDNA/percent-encoding paths. There is no standalone JSON resource for it upstream, so the array literal at the top of WPT's url/urlencoded-parser.any.js is hand-extracted and vendored as strict JSON under tools/urlencoded/urlencoded-parser.json, keeping only the plain input/output pairs that feed its "URLSearchParams constructed with" subtest -- the sibling formData() subtests exercise non-UTF-8 charsets kuri doesn't implement. Add a new Go generator (urlencoded.go) that reads that corpus and materializes a Kotlin fixture, following the same chunked-builder shape as the other WPT-backed generators, and wire it into the codegen dispatch table and the generateFixtures Gradle task. --- build.gradle.kts | 5 + tools/internal/codegen/main.go | 11 +- tools/internal/codegen/urlencoded.go | 223 +++++++++++++++ tools/urlencoded/urlencoded-parser.json | 352 ++++++++++++++++++++++++ 4 files changed, 588 insertions(+), 3 deletions(-) create mode 100644 tools/internal/codegen/urlencoded.go create mode 100644 tools/urlencoded/urlencoded-parser.json diff --git a/build.gradle.kts b/build.gradle.kts index 52d7caa..eba5802 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -75,6 +75,11 @@ val generators: List> = "percent-encoding", "Regenerate the WPT percent-encoding conformance fixture from the vendored ada WPT corpus.", ), + Triple( + "generateUrlEncodedTestData", + "urlencoded", + "Regenerate the WPT urlencoded-parser conformance fixture from tools/urlencoded/urlencoded-parser.json.", + ), ) val codegenTasks: List> = diff --git a/tools/internal/codegen/main.go b/tools/internal/codegen/main.go index 51dad6f..d40aafa 100644 --- a/tools/internal/codegen/main.go +++ b/tools/internal/codegen/main.go @@ -17,7 +17,8 @@ import ( // usage describes the command line for error messages. const usage = "usage: codegen [--stdout]\n" + - " names: url, idna-mapping, idna-validity, nfc-tables, nfc-test, conformance, setters, percent-encoding\n" + + " names: url, idna-mapping, idna-validity, nfc-tables, nfc-test, conformance, setters, percent-encoding,\n" + + " urlencoded\n" + " conformance also accepts --stdout-data / --stdout-known to print one of its two files" // singleGenerator pairs a single-file generator's source builder with a resolver @@ -27,8 +28,8 @@ type singleGenerator struct { output func() (string, error) } -// singleGenerators is the dispatch table for the five generators that emit one -// fixture each. conformance is handled separately in Main because it emits two +// singleGenerators is the dispatch table for the generators that each emit one +// fixture. conformance is handled separately in Main because it emits two // files and accepts its own stdout selectors. var singleGenerators = map[string]singleGenerator{ "url": { @@ -59,6 +60,10 @@ var singleGenerators = map[string]singleGenerator{ generate: generatePercentEncoding, output: outputPath("kuri", "src", "commonTest", "kotlin", "org", "dexpace", "kuri", "percent", "PercentEncodingTestData.kt"), }, + "urlencoded": { + generate: generateUrlEncoded, + output: outputPath("kuri", "src", "commonTest", "kotlin", "org", "dexpace", "kuri", "query", "UrlEncodedTestData.kt"), + }, } // Main dispatches to the named generator. The generator name is the first diff --git a/tools/internal/codegen/urlencoded.go b/tools/internal/codegen/urlencoded.go new file mode 100644 index 0000000..4e96504 --- /dev/null +++ b/tools/internal/codegen/urlencoded.go @@ -0,0 +1,223 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah +// SPDX-License-Identifier: MIT + +// The urlencoded generator reads the vendored WPT urlencoded-parser corpus and materializes the +// Kotlin UrlEncodedTestData fixture. There is no standalone JSON resource upstream: the corpus is +// hand-extracted from the array literal at the top of WPT's url/urlencoded-parser.any.js. Only the +// plain input/output pairs feeding that file's "URLSearchParams constructed with" subtest are +// vendored; its two sibling formData() subtests exercise non-UTF-8 charsets (windows-1252, +// shift_jis) that are out of scope, since kuri's form codec is UTF-8-only. Like percentencoding.go, +// no entry carries a lone surrogate, so decoding goes straight through encoding/json rather than +// url.go's/json.go's UTF-16 lossless path. +package codegen + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "unicode/utf16" +) + +// urlEncodedChunkSize is the number of cases per generated builder method, kept well under the +// 64 KB JVM constant-pool limit so no single method overflows. +const urlEncodedChunkSize = 60 + +// urlEncodedPair is one decoded name/value pair of a case's expected output, in appearance order. +type urlEncodedPair struct { + name string + value string +} + +// urlEncodedCase is one in-scope corpus entry: parsing input as application/x-www-form-urlencoded +// must yield output, in order. +type urlEncodedCase struct { + input string + output []urlEncodedPair +} + +// urlEncodedEntry mirrors one array element of urlencoded-parser.json: {input, output: [[name, +// value], ...]}. +type urlEncodedEntry struct { + Input *string `json:"input"` + Output [][]string `json:"output"` +} + +// urlEncodedInputPath returns the absolute path of the vendored WPT corpus. +func urlEncodedInputPath() (string, error) { + root, err := Root() + if err != nil { + return "", err + } + return filepath.Join(root, "tools", "urlencoded", "urlencoded-parser.json"), nil +} + +// generateUrlEncoded reads the corpus and returns the complete Kotlin source string. +func generateUrlEncoded() (string, error) { + path, err := urlEncodedInputPath() + if err != nil { + return "", err + } + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + cases, err := urlEncodedLoadCases(data) + if err != nil { + return "", err + } + return urlEncodedEmitFixture(cases) +} + +// urlEncodedLoadCases decodes the top-level JSON array and keeps only objects carrying an `input` +// key, preserving document order with no sorting or deduplication. The leading documentation +// string (skipped via firstNonSpace, as url.go/percentencoding.go do) is dropped. +func urlEncodedLoadCases(data []byte) ([]urlEncodedCase, error) { + var elements []json.RawMessage + if err := json.Unmarshal(data, &elements); err != nil { + return nil, fmt.Errorf("urlencoded: decoding corpus array: %w", err) + } + cases := make([]urlEncodedCase, 0, len(elements)) + for index, element := range elements { + if firstNonSpace(element) != '{' { + continue // documentation string + } + var entry urlEncodedEntry + if err := json.Unmarshal(element, &entry); err != nil { + return nil, fmt.Errorf("urlencoded: decoding object at index %d: %w", index, err) + } + if entry.Input == nil { + continue + } + output, err := urlEncodedPairs(entry.Output, index) + if err != nil { + return nil, err + } + cases = append(cases, urlEncodedCase{input: *entry.Input, output: output}) + } + return cases, nil +} + +// urlEncodedPairs converts one entry's raw [[name, value], ...] rows into pairs, erroring on any +// row that is not exactly two elements (the shape every corpus row currently has). +func urlEncodedPairs(rows [][]string, index int) ([]urlEncodedPair, error) { + pairs := make([]urlEncodedPair, 0, len(rows)) + for _, row := range rows { + if len(row) != 2 { + return nil, fmt.Errorf("urlencoded: entry %d has an output row with %d elements, want 2", index, len(row)) + } + pairs = append(pairs, urlEncodedPair{name: row[0], value: row[1]}) + } + return pairs, nil +} + +// urlEncodedEmitFixture renders the complete fixture: license header, suppress block, package, +// data class, and the chunked builder object, terminated by exactly one newline. The structure +// mirrors urlEmitFixture/percentEmitFixture. +func urlEncodedEmitFixture(cases []urlEncodedCase) (string, error) { + parts := Chunk(cases, urlEncodedChunkSize) + lines := []string{LicenseHeader, ""} + lines = append(lines, FileSuppressBlock(bulkDataComments, "LongMethod", "LargeClass", "MatchingDeclarationName")...) + lines = append(lines, + "package org.dexpace.kuri.query", + "", + ) + lines = append(lines, urlEncodedDataClassLines()...) + lines = append(lines, + "", + "// Generated by ./gradlew generateUrlEncodedTestData from the WPT urlencoded-parser corpus.", + "// Chunked into small builders so no single method exceeds the 64 KB JVM limit.", + "private object UrlEncodedCaseData {", + ) + lines = append(lines, urlEncodedAllAccessorLines(parts)...) + for index, part := range parts { + lines = append(lines, "") + lines = append(lines, fmt.Sprintf(" private fun part%d(): List =", index)) + lines = append(lines, " listOf(") + for _, current := range part { + caseLines, err := urlEncodedCaseLines(current) + if err != nil { + return "", err + } + lines = append(lines, caseLines...) + } + lines = append(lines, " )") + } + lines = append(lines, + "}", + "", + "/** Every WPT `urlencoded-parser.json` case (objects carrying an `input`). */", + "internal val URL_ENCODED_TEST_CASES: List = UrlEncodedCaseData.all()", + ) + return JoinLines(lines), nil +} + +// urlEncodedAllAccessorLines renders the `fun all(): List = ...` accessor, +// mirroring percentencoding.go's allAccessorLines: with a single chunk (the whole corpus is a +// handful of cases) PartSumExpression collapses to one term and, when the combined signature and +// body still fit MaxCols, ktlint's function-signature rule requires that single line rather than a +// wrap; with more than one chunk the body spans multiple lines and must wrap regardless. +func urlEncodedAllAccessorLines(parts [][]urlEncodedCase) []string { + signature := " fun all(): List =" + body := PartSumExpression(len(parts)) + if !strings.Contains(body, "\n") { + combined := signature + " " + body + if len(combined) <= MaxCols { + return []string{combined} + } + } + return []string{signature, " " + body} +} + +// urlEncodedCaseLines renders one UrlEncodedCase constructor call as indented named arguments. +func urlEncodedCaseLines(c urlEncodedCase) ([]string, error) { + lines := []string{" UrlEncodedCase("} + lines = append(lines, FieldLines("input", utf16.Encode([]rune(c.input)), fieldIndent)...) + outputLines, err := urlEncodedOutputLines(c.output) + if err != nil { + return nil, err + } + lines = append(lines, outputLines...) + lines = append(lines, " ),") + return lines, nil +} + +// urlEncodedOutputLines renders the `output = listOf("name" to "value", ...),` field (or +// `output = emptyList(),` when the case yields no pairs). ktlint requires a multiline call on the +// right of an `=` to start on its own line (as expectedFieldLines notes for `mapOf`), so `listOf(` +// cannot trail `output =` on the same line. Every corpus name/value is short enough that one pair +// fits one line comfortably within MaxCols; a pair that would not is a genuine generator gap, so it +// errors rather than silently emitting an over-long line for ktlint to catch. +func urlEncodedOutputLines(pairs []urlEncodedPair) ([]string, error) { + pad := strings.Repeat(" ", fieldIndent) + if len(pairs) == 0 { + return []string{pad + "output = emptyList(),"}, nil + } + listPad := strings.Repeat(" ", fieldIndent+4) + entryPad := strings.Repeat(" ", fieldIndent+8) + lines := []string{pad + "output =", listPad + "listOf("} + for _, pair := range pairs { + line := entryPad + KotlinString(pair.name) + " to " + KotlinString(pair.value) + "," + if len(line) > MaxCols { + return nil, fmt.Errorf("urlencoded: output pair %q/%q exceeds %d columns", pair.name, pair.value, MaxCols) + } + lines = append(lines, line) + } + lines = append(lines, listPad+"),") + return lines, nil +} + +// urlEncodedDataClassLines is the fixed KDoc + UrlEncodedCase data-class declaration. +func urlEncodedDataClassLines() []string { + return []string{ + "/**", + " * One WPT `urlencoded-parser.json` case: parsing [input] as `application/x-www-form-urlencoded`", + " * must yield [output], the ordered list of decoded `(name, value)` pairs.", + " */", + "internal data class UrlEncodedCase(", + " val input: String,", + " val output: List>,", + ")", + } +} diff --git a/tools/urlencoded/urlencoded-parser.json b/tools/urlencoded/urlencoded-parser.json new file mode 100644 index 0000000..39b4209 --- /dev/null +++ b/tools/urlencoded/urlencoded-parser.json @@ -0,0 +1,352 @@ +[ + "Vendored from the WPT array literal at the top of url/urlencoded-parser.any.js (web-platform-tests/wpt, master branch, fetched 2026-07-13). Only the plain input/output pairs used by the \"URLSearchParams constructed with\" subtest are kept; the two per-entry formData() subtests exercise non-UTF-8 charsets (windows-1252, shift_jis) that are out of scope for kuri, which is UTF-8-only.", + { + "input": "test", + "output": [ + [ + "test", + "" + ] + ] + }, + { + "input": "test=", + "output": [ + [ + "test", + "" + ] + ] + }, + { + "input": "%EF%BB%BFtest=%EF%BB%BF", + "output": [ + [ + "test", + "" + ] + ] + }, + { + "input": "%EF%BF%BF=%EF%BF%BF", + "output": [ + [ + "￿", + "￿" + ] + ] + }, + { + "input": "%FE%FF", + "output": [ + [ + "��", + "" + ] + ] + }, + { + "input": "%FF%FE", + "output": [ + [ + "��", + "" + ] + ] + }, + { + "input": "†&†=x", + "output": [ + [ + "†", + "" + ], + [ + "†", + "x" + ] + ] + }, + { + "input": "%C2", + "output": [ + [ + "�", + "" + ] + ] + }, + { + "input": "%C2x", + "output": [ + [ + "�x", + "" + ] + ] + }, + { + "input": "_charset_=windows-1252&test=%C2x", + "output": [ + [ + "_charset_", + "windows-1252" + ], + [ + "test", + "�x" + ] + ] + }, + { + "input": "", + "output": [] + }, + { + "input": "a", + "output": [ + [ + "a", + "" + ] + ] + }, + { + "input": "a=b", + "output": [ + [ + "a", + "b" + ] + ] + }, + { + "input": "a=", + "output": [ + [ + "a", + "" + ] + ] + }, + { + "input": "=b", + "output": [ + [ + "", + "b" + ] + ] + }, + { + "input": "&", + "output": [] + }, + { + "input": "&a", + "output": [ + [ + "a", + "" + ] + ] + }, + { + "input": "a&", + "output": [ + [ + "a", + "" + ] + ] + }, + { + "input": "a&a", + "output": [ + [ + "a", + "" + ], + [ + "a", + "" + ] + ] + }, + { + "input": "a&b&c", + "output": [ + [ + "a", + "" + ], + [ + "b", + "" + ], + [ + "c", + "" + ] + ] + }, + { + "input": "a=b&c=d", + "output": [ + [ + "a", + "b" + ], + [ + "c", + "d" + ] + ] + }, + { + "input": "a=b&c=d&", + "output": [ + [ + "a", + "b" + ], + [ + "c", + "d" + ] + ] + }, + { + "input": "&&&a=b&&&&c=d&", + "output": [ + [ + "a", + "b" + ], + [ + "c", + "d" + ] + ] + }, + { + "input": "a=a&a=b&a=c", + "output": [ + [ + "a", + "a" + ], + [ + "a", + "b" + ], + [ + "a", + "c" + ] + ] + }, + { + "input": "a==a", + "output": [ + [ + "a", + "=a" + ] + ] + }, + { + "input": "a=a+b+c+d", + "output": [ + [ + "a", + "a b c d" + ] + ] + }, + { + "input": "%=a", + "output": [ + [ + "%", + "a" + ] + ] + }, + { + "input": "%a=a", + "output": [ + [ + "%a", + "a" + ] + ] + }, + { + "input": "%a_=a", + "output": [ + [ + "%a_", + "a" + ] + ] + }, + { + "input": "%61=a", + "output": [ + [ + "a", + "a" + ] + ] + }, + { + "input": "%61+%4d%4D=", + "output": [ + [ + "a MM", + "" + ] + ] + }, + { + "input": "id=0&value=%", + "output": [ + [ + "id", + "0" + ], + [ + "value", + "%" + ] + ] + }, + { + "input": "b=%2sf%2a", + "output": [ + [ + "b", + "%2sf*" + ] + ] + }, + { + "input": "b=%2%2af%2a", + "output": [ + [ + "b", + "%2*f*" + ] + ] + }, + { + "input": "b=%%2a", + "output": [ + [ + "b", + "%*" + ] + ] + } +] From cf693d90f5ea728fa4c117c2eee5bb4975a33f28 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:36:53 +0300 Subject: [PATCH 08/13] test: add a ratcheting conformance suite for form-urlencoded parsing Generate the UrlEncodedTestData fixture from the vendored WPT corpus and run every case through FormUrlEncoded.parse, mirroring the ratchet structure the percent-encoding and IDNA suites already use. All 35 in-scope cases pass against the existing implementation, so the tracked known-failures baseline is empty. --- .../kuri/query/UrlEncodedConformanceTest.kt | 74 +++++ .../dexpace/kuri/query/UrlEncodedTestData.kt | 282 ++++++++++++++++++ 2 files changed, 356 insertions(+) create mode 100644 kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedConformanceTest.kt create mode 100644 kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedTestData.kt diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedConformanceTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedConformanceTest.kt new file mode 100644 index 0000000..e433c68 --- /dev/null +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedConformanceTest.kt @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2026 dexpace and Omar Aljarrah + * SPDX-License-Identifier: MIT + */ + +package org.dexpace.kuri.query + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Runs every WPT `urlencoded-parser.json` case (modelled as [URL_ENCODED_TEST_CASES]) through + * [FormUrlEncoded.parse] and ratchets a tracked known-failures baseline (SPEC §10.4, [QUERY-21]). + * + * The corpus is the array literal WPT's `url/urlencoded-parser.any.js` feeds to its + * "URLSearchParams constructed with" subtest: a flat `{input, output: [[name, value], ...]}` + * shape with no `failure` field, so [FormUrlEncoded.parse]'s return type + * (`List>`) already matches [UrlEncodedCase.output] with no adaptation. The + * file's two sibling per-entry subtests (`request.formData()`/`response.formData()`) exercise + * non-UTF-8 charsets (`windows-1252`, `shift_jis`) and are out of scope, since kuri's form codec + * is UTF-8-only; they are not vendored. + * + * There is no residual divergence: [KNOWN_FAILURES] is empty and the suite is at full + * conformance. It still ratchets in both directions -- a brand-new failure breaks the + * untracked-regressions test, and a regression that repopulates the live set breaks the + * baseline-equality test until the baseline is updated. + */ +class UrlEncodedConformanceTest { + /** True when parsing [case]'s input reproduces its expected output pairs, in order. */ + private fun caseMatches(case: UrlEncodedCase): Boolean = FormUrlEncoded.parse(case.input) == case.output + + /** The inputs of every case the live parser currently does not satisfy. */ + private fun liveFailingInputs(): Set = + URL_ENCODED_TEST_CASES.filterNot { caseMatches(it) }.map { it.input }.toSet() + + @Test + fun `every urlencoded case outside the known-failures set matches WPT`() { + val untracked = liveFailingInputs() - KNOWN_FAILURES + assertTrue(untracked.isEmpty(), "new urlencoded-parser regressions (untracked failures): $untracked") + } + + @Test + fun `the known-failures set exactly equals the live failing set`() { + // Ratchet: a fixed gap (tracked failure now passing) or a brand-new failure breaks this + // until the baseline is regenerated, so the deferred debt can never drift silently. + assertEquals(KNOWN_FAILURES, liveFailingInputs()) + } + + @Test + fun `every tracked known failure is a real corpus case`() { + val inputs = URL_ENCODED_TEST_CASES.map { it.input }.toSet() + val orphans = KNOWN_FAILURES - inputs + assertTrue(orphans.isEmpty(), "known failures absent from the corpus: $orphans") + } + + @Test + fun `the corpus is non-empty and fully conformant`() { + assertTrue(URL_ENCODED_TEST_CASES.isNotEmpty(), "the WPT corpus should not be empty") + val failing = liveFailingInputs() + assertTrue(failing.isEmpty(), "every WPT case must parse per spec; failing: $failing") + } + + private companion object { + /** + * The tracked baseline of currently-failing case inputs. It is empty: the live form parser + * satisfies every WPT `urlencoded-parser.json` case. The ratcheting + * `the known-failures set exactly equals the live failing set` test pins the suite at full + * conformance: any regression repopulates the live set and breaks the build until this + * baseline is updated. + */ + private val KNOWN_FAILURES: Set = emptySet() + } +} diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedTestData.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedTestData.kt new file mode 100644 index 0000000..c5b0cce --- /dev/null +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedTestData.kt @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2026 dexpace and Omar Aljarrah + * SPDX-License-Identifier: MIT + */ + +// Generated bulk data, not hand-written logic: the chunked builders intentionally exceed +// detekt's method/class-size heuristics to stay within the 64 KB JVM method limit. +@file:Suppress("LongMethod", "LargeClass", "MatchingDeclarationName") + +package org.dexpace.kuri.query + +/** + * One WPT `urlencoded-parser.json` case: parsing [input] as `application/x-www-form-urlencoded` + * must yield [output], the ordered list of decoded `(name, value)` pairs. + */ +internal data class UrlEncodedCase( + val input: String, + val output: List>, +) + +// Generated by ./gradlew generateUrlEncodedTestData from the WPT urlencoded-parser corpus. +// Chunked into small builders so no single method exceeds the 64 KB JVM limit. +private object UrlEncodedCaseData { + fun all(): List = part0() + + private fun part0(): List = + listOf( + UrlEncodedCase( + input = "test", + output = + listOf( + "test" to "", + ), + ), + UrlEncodedCase( + input = "\ufefftest=\ufeff", + output = + listOf( + "\ufefftest" to "\ufeff", + ), + ), + UrlEncodedCase( + input = "%EF%BB%BFtest=%EF%BB%BF", + output = + listOf( + "\ufefftest" to "\ufeff", + ), + ), + UrlEncodedCase( + input = "%EF%BF%BF=%EF%BF%BF", + output = + listOf( + "\uffff" to "\uffff", + ), + ), + UrlEncodedCase( + input = "%FE%FF", + output = + listOf( + "\ufffd\ufffd" to "", + ), + ), + UrlEncodedCase( + input = "%FF%FE", + output = + listOf( + "\ufffd\ufffd" to "", + ), + ), + UrlEncodedCase( + input = "\u2020&\u2020=x", + output = + listOf( + "\u2020" to "", + "\u2020" to "x", + ), + ), + UrlEncodedCase( + input = "%C2", + output = + listOf( + "\ufffd" to "", + ), + ), + UrlEncodedCase( + input = "%C2x", + output = + listOf( + "\ufffdx" to "", + ), + ), + UrlEncodedCase( + input = "_charset_=windows-1252&test=%C2x", + output = + listOf( + "_charset_" to "windows-1252", + "test" to "\ufffdx", + ), + ), + UrlEncodedCase( + input = "", + output = emptyList(), + ), + UrlEncodedCase( + input = "a", + output = + listOf( + "a" to "", + ), + ), + UrlEncodedCase( + input = "a=b", + output = + listOf( + "a" to "b", + ), + ), + UrlEncodedCase( + input = "a=", + output = + listOf( + "a" to "", + ), + ), + UrlEncodedCase( + input = "=b", + output = + listOf( + "" to "b", + ), + ), + UrlEncodedCase( + input = "&", + output = emptyList(), + ), + UrlEncodedCase( + input = "&a", + output = + listOf( + "a" to "", + ), + ), + UrlEncodedCase( + input = "a&", + output = + listOf( + "a" to "", + ), + ), + UrlEncodedCase( + input = "a&a", + output = + listOf( + "a" to "", + "a" to "", + ), + ), + UrlEncodedCase( + input = "a&b&c", + output = + listOf( + "a" to "", + "b" to "", + "c" to "", + ), + ), + UrlEncodedCase( + input = "a=b&c=d", + output = + listOf( + "a" to "b", + "c" to "d", + ), + ), + UrlEncodedCase( + input = "a=b&c=d&", + output = + listOf( + "a" to "b", + "c" to "d", + ), + ), + UrlEncodedCase( + input = "&&&a=b&&&&c=d&", + output = + listOf( + "a" to "b", + "c" to "d", + ), + ), + UrlEncodedCase( + input = "a=a&a=b&a=c", + output = + listOf( + "a" to "a", + "a" to "b", + "a" to "c", + ), + ), + UrlEncodedCase( + input = "a==a", + output = + listOf( + "a" to "=a", + ), + ), + UrlEncodedCase( + input = "a=a+b+c+d", + output = + listOf( + "a" to "a b c d", + ), + ), + UrlEncodedCase( + input = "%=a", + output = + listOf( + "%" to "a", + ), + ), + UrlEncodedCase( + input = "%a=a", + output = + listOf( + "%a" to "a", + ), + ), + UrlEncodedCase( + input = "%a_=a", + output = + listOf( + "%a_" to "a", + ), + ), + UrlEncodedCase( + input = "%61=a", + output = + listOf( + "a" to "a", + ), + ), + UrlEncodedCase( + input = "%61+%4d%4D=", + output = + listOf( + "a MM" to "", + ), + ), + UrlEncodedCase( + input = "id=0&value=%", + output = + listOf( + "id" to "0", + "value" to "%", + ), + ), + UrlEncodedCase( + input = "b=%2sf%2a", + output = + listOf( + "b" to "%2sf*", + ), + ), + UrlEncodedCase( + input = "b=%2%2af%2a", + output = + listOf( + "b" to "%2*f*", + ), + ), + UrlEncodedCase( + input = "b=%%2a", + output = + listOf( + "b" to "%*", + ), + ), + ) +} + +/** Every WPT `urlencoded-parser.json` case (objects carrying an `input`). */ +internal val URL_ENCODED_TEST_CASES: List = UrlEncodedCaseData.all() From f6bb3f0c3f108793e00f0a1a35e3f5068051845e Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:36:57 +0300 Subject: [PATCH 09/13] docs: mark form-urlencoded as Conformant now that it's measured The parsing/serialization path is now checked against the WPT urlencoded-parser corpus with no known failures, so it earns the same Conformant label the other measured standards carry, and the new suite's result is listed in the Conformance table. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 62ab693..5187471 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ directionality restriction, which the RFC itself states as a SHOULD, not a MUST. | Standard | Governs | Compliance | Support | |------------------------------------------------|----------------------------------------------|------------|---------| -| [`application/x-www-form-urlencoded`][formenc] | Form-encoded query parsing and serialization | Supported | Default | +| [`application/x-www-form-urlencoded`][formenc] | Form-encoded query parsing and serialization | Conformant | Default | **Notation and requirement levels** @@ -550,6 +550,7 @@ Behavior is checked against the conformance corpora the standards ship with: | IDNA `IdnaTestV2` + `toascii` | 2756 / 2760 | | Unicode `NormalizationTest.txt` (NFC) | 20 034 / 20 034 | | RFC 3986 §5.4 reference resolution | all rows | +| WHATWG `urlencoded-parser.any.js` — form parsing | 35 / 35 | Any case that does not yet pass is pinned in a checked-in known-failures baseline; the build fails if a passing case later regresses. From 94c13c50114c896cb358af1fbfc6c1254da17f97 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 19:59:48 +0300 Subject: [PATCH 10/13] fix: make the ASCII/non-ASCII leniency decide on the raw pre-mapping label The per-label ASCII leniency in the Url domain-to-ASCII path mapped the whole domain before splitting into labels and checking each label's ASCII-ness. That let a non-ASCII sequence that renders down to a literal ASCII string under UTS-46 mapping (e.g. a fullwidth "xn--a") take the unconditional-keep branch and skip the Punycode decode/validate step entirely, even though the direct domainToAscii pipeline still rejects it. Split the raw domain on the literal dot separator and its three non-ASCII UTS-46 variants instead of mapping first, and make the ASCII/non-ASCII decision on each raw label before any mapping runs. This keeps the label boundary a mapped dot variant forms visible without letting mapped content leak into the leniency check. --- .../kotlin/org/dexpace/kuri/idna/Idna.kt | 63 +++++++++++++------ .../kotlin/org/dexpace/kuri/idna/IdnTest.kt | 12 ++++ 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt index 204be02..44c8d23 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt @@ -24,6 +24,18 @@ private const val ACE_PREFIX_LENGTH: Int = 4 /** Label separator inside a domain (U+002E FULL STOP); also the join separator. */ private const val LABEL_SEPARATOR: String = "." +/** [LABEL_SEPARATOR] as a `Char`, for splitting a raw (pre-mapping) domain by code point. */ +private const val LABEL_SEPARATOR_CHAR: Char = '.' + +/** U+3002 IDEOGRAPHIC FULL STOP: a UTS-46 dot-separator variant [IdnaMapping.Mapped] to `.`. */ +private const val IDEOGRAPHIC_FULL_STOP: Char = '\u3002' + +/** U+FF0E FULLWIDTH FULL STOP: a UTS-46 dot-separator variant [IdnaMapping.Mapped] to `.`. */ +private const val FULLWIDTH_FULL_STOP: Char = '\uFF0E' + +/** U+FF61 HALFWIDTH IDEOGRAPHIC FULL STOP: a UTS-46 dot-separator variant [IdnaMapping.Mapped] to `.`. */ +private const val HALFWIDTH_IDEOGRAPHIC_FULL_STOP: Char = '\uFF61' + /** * UTS-46 ToASCII / ToUnicode for IDNA domains (SPEC §7.4, [HOST-26]) under the `Url`-profile * parameter set ([HOST-28]): `CheckHyphens = false`, `CheckBidi = true`, `CheckJoiners = true`, @@ -91,27 +103,29 @@ internal object Idna { } /** - * Maps [domain] first, then splits the mapped-and-NFC-normalized text on [LABEL_SEPARATOR] and - * resolves each label independently: an all-ASCII label is lowercased and kept unconditionally - * (never validated, matching the existing intentional leniency — see [domainToAsciiForUrl]); a - * label carrying any non-ASCII code point runs the full [domainToAscii] pipeline, and that - * label's failure fails the whole domain. Mapping before splitting (mirroring [domainToAscii]'s - * own step order) is required so a label boundary formed by one of the three non-ASCII UTS-46 - * dot-separator variants (U+3002, U+FF0E, U+FF61 — all [IdnaMapping.Mapped] to [LABEL_SEPARATOR]) - * is visible here too, not only inside the nested [domainToAscii] call on a non-ASCII label. - * Mapping the whole domain up front is safe: every ASCII code point reaching this function - * already passed the WHATWG forbidden-host-code-point filter upstream, and no ASCII code point - * maps outside ASCII or is dropped under `UseSTD3ASCIIRules = false`, so an all-ASCII label's - * content is unchanged in substance (only case-folded, which [asciiLowercase] already redoes - * harmlessly) and a `mapAll` failure can only occur where the nested [domainToAscii] on that same - * span would already have failed today. + * Splits the raw (pre-mapping) [domain] on [LABEL_SEPARATOR] or any of the three non-ASCII + * UTS-46 dot-separator variants (see [splitRawLabels]) and resolves each raw label + * independently: a label that is already all-ASCII **before any mapping runs** is lowercased + * and kept unconditionally (never validated, matching the existing intentional leniency — see + * [domainToAsciiForUrl]); a label carrying any non-ASCII code point runs the full + * [domainToAscii] pipeline (which performs the mapping, NFC, and Punycode decode/validate steps + * itself), and that label's failure fails the whole domain. + * + * The ASCII/non-ASCII decision is deliberately made on each **raw** label, not on a + * whole-domain-mapped one: mapping the domain before this decision would let a non-ASCII + * sequence that maps down to a literal ASCII string (e.g. a fullwidth rendering of `xn--a`) + * take the unconditional-keep branch and skip [domainToAscii]'s Punycode decode/validate step + * entirely — exactly the case this leniency must not cover, since HOST-48 scopes it to a + * domain that is already ASCII pre-mapping. Splitting on the raw dot-separator variants (rather + * than only [LABEL_SEPARATOR]) keeps the boundary that the mapping step would otherwise reveal + * visible here too, without needing to map first: those three code points are always mapped to + * exactly [LABEL_SEPARATOR] on their own (never merged into a longer replacement), so a literal + * split on them is equivalent to mapping-then-splitting for boundary purposes alone. * - * @return the joined ASCII domain, or `null` on a mapping failure or the first non-ASCII label's - * UTS-46 failure. + * @return the joined ASCII domain, or `null` on the first non-ASCII label's UTS-46 failure. */ private fun asciiLenientDomainToAscii(domain: String): String? { - val mapped = mapAll(domain) ?: return null - val labels = splitLabels(Normalizer.nfc(mapped)) + val labels = splitRawLabels(domain) val out = ArrayList(labels.size) var index = 0 var failed = false @@ -187,6 +201,19 @@ internal object Idna { /** Splits [domain] into labels on [LABEL_SEPARATOR]; an empty domain yields one empty label. */ private fun splitLabels(domain: String): List = domain.split(LABEL_SEPARATOR) + /** + * Splits the raw, pre-mapping [domain] into labels on [LABEL_SEPARATOR] or any of the three + * non-ASCII UTS-46 dot-separator variants that map to it, so a label boundary they form is + * visible before mapping runs (see [asciiLenientDomainToAscii]). + */ + private fun splitRawLabels(domain: String): List = + domain.split( + LABEL_SEPARATOR_CHAR, + IDEOGRAPHIC_FULL_STOP, + FULLWIDTH_FULL_STOP, + HALFWIDTH_IDEOGRAPHIC_FULL_STOP, + ) + /** * Processes every label in order, short-circuiting on the first failure so the resulting * [ParseResult.Err] carries the original [domain] for diagnostics ([HOST-26] steps 4 & 5). diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt index db67239..7bb4377 100644 --- a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt @@ -82,6 +82,18 @@ class IdnTest { assertEquals(expected.getOrNull(), Idn.toAscii(input).getOrNull()) } + @Test + fun `toAscii rejects a fullwidth label that maps down to a literal xn-- ascii string`() { + // U+FF58 U+FF4E U+FF0D U+FF0D 'a' (fullwidth x, n, hyphen-minus x2, then ascii 'a') maps + // down to the literal ASCII text "xn--a" under UTS-46 mapping. The per-label ASCII/ + // non-ASCII routing decision must be made on this original non-ASCII label, not on its + // post-mapping text, or the label slips past the Punycode decode/validate step that + // Idna.domainToAscii("xn--a") itself fails. + val fullwidthLabel = Char(0xFF58).toString() + Char(0xFF4E) + Char(0xFF0D) + Char(0xFF0D) + "a" + + assertFalse(Idn.toAscii(fullwidthLabel).isOk()) + } + @Test fun `toAscii fails on a disallowed code point`() { // U+0080 is a plain disallowed code point, so ToASCII must reject it fatally. From 79bd19662b95fb2c534b4e27d466444a67193053 Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 20:00:01 +0300 Subject: [PATCH 11/13] build: source the android BCV runtime classpath's ASM version from the catalog The hand-wired android apiBuild/apiCheck classpath pinned org.ow2.asm:asm and asm-tree with literal version strings, bypassing gradle/libs.versions.toml as the single source of truth for dependency versions. Add an asm entry to the catalog and reference it from build.gradle.kts so a future ASM bump has one place to update and shows up to Dependabot. --- gradle/libs.versions.toml | 7 +++++++ kuri/build.gradle.kts | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 401f4c5..89139fc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,6 +8,10 @@ binary-compat = "0.18.1" dokka = "2.0.0" maven-publish = "0.30.0" icu4j = "77.1" +# Pinned to match the ASM version binary-compatibility-validator 0.18.1 wires into its own +# jvm/native apiBuild classpath internally, so the hand-wired android apiBuild leg (build.gradle.kts) +# parses class files with the same ASM release the rest of the apiCheck gate uses. +asm = "9.6" [libraries] # kotlin-test is pulled in via the kotlin("test") MPP helper; listed for reference. @@ -16,6 +20,9 @@ kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = # engine against (see IdnaIcu4jDifferentialTest). No multiplatform artifact exists, so this # must never be referenced from commonMain/commonTest. icu4j = { module = "com.ibm.icu:icu4j", version.ref = "icu4j" } +# Runtime classpath for the manually wired android apiBuild/apiCheck tasks (build.gradle.kts). +asm-core = { module = "org.ow2.asm:asm", version.ref = "asm" } +asm-tree = { module = "org.ow2.asm:asm-tree", version.ref = "asm" } [plugins] kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } diff --git a/kuri/build.gradle.kts b/kuri/build.gradle.kts index 0bcc562..ea08412 100644 --- a/kuri/build.gradle.kts +++ b/kuri/build.gradle.kts @@ -270,8 +270,8 @@ val androidAbiRuntimeClasspath: NamedDomainObjectProvider = } dependencies.apply { - add(androidAbiRuntimeClasspath.name, "org.ow2.asm:asm:9.6") - add(androidAbiRuntimeClasspath.name, "org.ow2.asm:asm-tree:9.6") + add(androidAbiRuntimeClasspath.name, libs.asm.core.get()) + add(androidAbiRuntimeClasspath.name, libs.asm.tree.get()) add(androidAbiRuntimeClasspath.name, "org.jetbrains.kotlin:kotlin-metadata-jvm:${libs.versions.kotlin.get()}") } From e750c2512d556300d533f9977f7ac66e39d6c89d Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 21:06:04 +0300 Subject: [PATCH 12/13] build: wire androidApiBuild.inputDependencies and move kotlin-metadata-jvm into the catalog androidApiBuild (the hand-wired android BCV task) never set inputDependencies, unlike every BCV-auto-configured target, which wires it from the compilation's compileDependencyFiles. Wire the same thing from the android target's main compilation for parity, and move the kotlin-metadata-jvm coordinate from an inline string into gradle/libs.versions.toml alongside asm-core/asm-tree, so every dependency in this classpath is declared the same way. --- gradle/libs.versions.toml | 1 + kuri/build.gradle.kts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 89139fc..09e57e6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,6 +23,7 @@ icu4j = { module = "com.ibm.icu:icu4j", version.ref = "icu4j" } # Runtime classpath for the manually wired android apiBuild/apiCheck tasks (build.gradle.kts). asm-core = { module = "org.ow2.asm:asm", version.ref = "asm" } asm-tree = { module = "org.ow2.asm:asm-tree", version.ref = "asm" } +kotlin-metadata-jvm = { module = "org.jetbrains.kotlin:kotlin-metadata-jvm", version.ref = "kotlin" } [plugins] kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } diff --git a/kuri/build.gradle.kts b/kuri/build.gradle.kts index ea08412..32520cd 100644 --- a/kuri/build.gradle.kts +++ b/kuri/build.gradle.kts @@ -272,7 +272,11 @@ val androidAbiRuntimeClasspath: NamedDomainObjectProvider = dependencies.apply { add(androidAbiRuntimeClasspath.name, libs.asm.core.get()) add(androidAbiRuntimeClasspath.name, libs.asm.tree.get()) - add(androidAbiRuntimeClasspath.name, "org.jetbrains.kotlin:kotlin-metadata-jvm:${libs.versions.kotlin.get()}") + add( + androidAbiRuntimeClasspath.name, + libs.kotlin.metadata.jvm + .get(), + ) } val androidApiBuild: TaskProvider = @@ -280,6 +284,13 @@ val androidApiBuild: TaskProvider = group = "verification" description = "Builds the Kotlin API dump for the android target's 'main' compilation of kuri." inputClassesDirs.from(tasks.named("compileAndroidMain").map { it.outputs.files }) + inputDependencies.from( + kotlin.targets + .getByName("android") + .compilations + .getByName("main") + .compileDependencyFiles, + ) outputApiFile.set(layout.buildDirectory.file("kotlin/abi-android/kuri.api")) runtimeClasspath.from(androidAbiRuntimeClasspath) } From 31bd70e97997bcea740c1b4f355a12b31067416f Mon Sep 17 00:00:00 2001 From: OmarAlJarrah Date: Mon, 13 Jul 2026 21:06:13 +0300 Subject: [PATCH 13/13] test: cover precedence between iri bidi, repertoire, and host-failure checks Iri.toUri runs a bidi-formatting-character scan over the whole raw IRI before the repertoire check, and returns either of those precheck failures before ever attempting to map the host. Neither ordering had a dedicated test: add one proving a bidi violation wins even when a repertoire violation appears earlier in the string, and one proving a precheck failure wins over an IDNA-invalid host. --- .../kotlin/org/dexpace/kuri/IriTest.kt | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt index 78fb7a9..f3c48e7 100644 --- a/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt @@ -162,6 +162,42 @@ class IriTest { assertEquals(iri.indexOf(lrm), error.at) } + @Test + fun `a bidi violation anywhere in the iri wins over an earlier repertoire violation`() { + // U+FDD0 is a repertoire violation (neither ucschar nor iprivate, see the noncharacter test + // above) placed BEFORE the LRM in raw text order. A naive single combined left-to-right scan + // would report the noncharacter first; the bidi scan is instead its own complete first pass + // over the whole raw string, so the later LRM wins the precedence, not just a leftmost match. + val nonCharacter = Char(0xFDD0).toString() + val lrm = Char(0x200E).toString() + val iri = "http://h/$nonCharacter$lrm" + + val error = + assertIs(assertIs(Iri.toUri(iri)).error) + + assertEquals(0x200E, error.codePoint) + assertEquals(iri.indexOf(lrm), error.at) + } + + @Test + fun `a precheck failure wins over an idna host failure`() { + // The leading combining mark makes this host idna-invalid on its own — see + // `fails when a non-ascii host is idna-invalid` above, which proves the same host text + // fails with UriParseError.InvalidHost(HostError.IdnaFailed) when used alone. Adding a + // repertoire-violating noncharacter to the path must still surface the precheck failure + // instead, since resolveHost short-circuits to the precheck error before ever mapping the + // host (see its KDoc). + val leadingCombiningMark = Char(0x0301).toString() + val nonCharacter = Char(0xFDD0).toString() + val invalidHost = "${leadingCombiningMark}x" + val iri = "http://$invalidHost/$nonCharacter" + + val error = assertIs(assertIs(Iri.toUri(iri)).error) + + assertEquals(0xFDD0, error.codePoint) + assertEquals(iri.indexOf(nonCharacter), error.at) + } + @Test fun `strict uri parse rejects a raw non-ascii host while the facility maps it`() { val iri = "http://$aUmlaut.example/"