diff --git a/README.md b/README.md index 23d3e41..5187471 100644 --- a/README.md +++ b/README.md @@ -473,7 +473,9 @@ kuri implements the standards below; per-standard conformance is measured in [Co | [RFC 3987][rfc3987] | IRIs — one-way `Iri` mapping, not a validating parser* | Supported | Default | | [WHATWG URL Standard][whatwg-url] | the `Url` model — parser, special schemes, canonical serialization | Conformant | Default | -\* kuri maps IRIs to URIs one-way (RFC 3987 §3.1/§3.2); it does not implement §4 validation (bidi, repertoire checks). +\* kuri maps IRIs to URIs one-way (RFC 3987 §3.1/§3.2), rejecting a §2.2 `ucschar`/`iprivate` +repertoire violation and a §4.1 bidi formatting character; it does not enforce §4.2's per-component +directionality restriction, which the RFC itself states as a SHOULD, not a MUST. **Hosts, internationalization, and IP addresses** @@ -491,7 +493,7 @@ kuri implements the standards below; per-standard conformance is measured in [Co | Standard | Governs | Compliance | Support | |------------------------------------------------|----------------------------------------------|------------|---------| -| [`application/x-www-form-urlencoded`][formenc] | Form-encoded query parsing and serialization | Supported | Default | +| [`application/x-www-form-urlencoded`][formenc] | Form-encoded query parsing and serialization | Conformant | Default | **Notation and requirement levels** @@ -548,6 +550,7 @@ Behavior is checked against the conformance corpora the standards ship with: | IDNA `IdnaTestV2` + `toascii` | 2756 / 2760 | | Unicode `NormalizationTest.txt` (NFC) | 20 034 / 20 034 | | RFC 3986 §5.4 reference resolution | all rows | +| WHATWG `urlencoded-parser.any.js` — form parsing | 35 / 35 | Any case that does not yet pass is pinned in a checked-in known-failures baseline; the build fails if a passing case later regresses. diff --git a/build.gradle.kts b/build.gradle.kts index 52d7caa..eba5802 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -75,6 +75,11 @@ val generators: List> = "percent-encoding", "Regenerate the WPT percent-encoding conformance fixture from the vendored ada WPT corpus.", ), + Triple( + "generateUrlEncodedTestData", + "urlencoded", + "Regenerate the WPT urlencoded-parser conformance fixture from tools/urlencoded/urlencoded-parser.json.", + ), ) val codegenTasks: List> = diff --git a/docs/idna-unicode-update.md b/docs/idna-unicode-update.md index 06e7a0e..ce0f7db 100644 --- a/docs/idna-unicode-update.md +++ b/docs/idna-unicode-update.md @@ -30,7 +30,9 @@ The generators are Go programs under `tools/` (module `github.com/dexpace/kuri/tools`), driven by `tools/internal/codegen`. That package reads raw UCD via `tools/internal/ucd` and derives the conformance baseline via `tools/internal/idnaref`, a faithful Go port of kuri's runtime IDNA -engine. +engine — a second implementation of kuri's own algorithm, not an independent +oracle. WPT (below) is the independent oracle; idnaref only narrows the +known-failures residual within whatever WPT doesn't already exercise. ## Source of truth diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a1d6fe5..09e57e6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,10 +7,23 @@ kover = "0.9.8" binary-compat = "0.18.1" dokka = "2.0.0" maven-publish = "0.30.0" +icu4j = "77.1" +# Pinned to match the ASM version binary-compatibility-validator 0.18.1 wires into its own +# jvm/native apiBuild classpath internally, so the hand-wired android apiBuild leg (build.gradle.kts) +# parses class files with the same ASM release the rest of the apiCheck gate uses. +asm = "9.6" [libraries] # kotlin-test is pulled in via the kotlin("test") MPP helper; listed for reference. kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" } +# jvmTest-only: an independent third-party UTS-46 implementation to cross-check kuri's IDNA +# engine against (see IdnaIcu4jDifferentialTest). No multiplatform artifact exists, so this +# must never be referenced from commonMain/commonTest. +icu4j = { module = "com.ibm.icu:icu4j", version.ref = "icu4j" } +# Runtime classpath for the manually wired android apiBuild/apiCheck tasks (build.gradle.kts). +asm-core = { module = "org.ow2.asm:asm", version.ref = "asm" } +asm-tree = { module = "org.ow2.asm:asm-tree", version.ref = "asm" } +kotlin-metadata-jvm = { module = "org.jetbrains.kotlin:kotlin-metadata-jvm", version.ref = "kotlin" } [plugins] kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } diff --git a/kuri/api/android/kuri.api b/kuri/api/android/kuri.api index fa8e1ff..87de2b6 100644 --- a/kuri/api/android/kuri.api +++ b/kuri/api/android/kuri.api @@ -362,6 +362,34 @@ public final class org/dexpace/kuri/error/UriParseError$InvalidScheme : org/dexp public fun toString ()Ljava/lang/String; } +public final class org/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter : org/dexpace/kuri/error/UriParseError { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter; + public static synthetic fun copy$default (Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter;IIILjava/lang/Object;)Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter; + public fun equals (Ljava/lang/Object;)Z + public final fun getAt ()I + public final fun getCodePoint ()I + public fun getMessage ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class org/dexpace/kuri/error/UriParseError$IriInvalidCodePoint : org/dexpace/kuri/error/UriParseError { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint; + public static synthetic fun copy$default (Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint;IIILjava/lang/Object;)Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint; + public fun equals (Ljava/lang/Object;)Z + public final fun getAt ()I + public final fun getCodePoint ()I + public fun getMessage ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class org/dexpace/kuri/error/UriParseError$MissingScheme : org/dexpace/kuri/error/UriParseError { public static final field INSTANCE Lorg/dexpace/kuri/error/UriParseError$MissingScheme; public fun equals (Ljava/lang/Object;)Z @@ -376,6 +404,7 @@ public final class org/dexpace/kuri/error/UriSyntaxException : java/lang/Illegal public final class org/dexpace/kuri/error/ValidationError : java/lang/Enum { public static final field BACKSLASH_AS_SOLIDUS Lorg/dexpace/kuri/error/ValidationError; + public static final field INVALID_CREDENTIALS Lorg/dexpace/kuri/error/ValidationError; public static final field INVALID_URL_UNIT Lorg/dexpace/kuri/error/ValidationError; public static final field LEADING_OR_TRAILING_C0_CONTROL_OR_SPACE Lorg/dexpace/kuri/error/ValidationError; public static final field MISSING_AUTHORITY_SLASHES Lorg/dexpace/kuri/error/ValidationError; @@ -519,6 +548,7 @@ public final class org/dexpace/kuri/query/QueryParameters : java/lang/Iterable, public static final fun parse (Ljava/lang/String;)Lorg/dexpace/kuri/query/QueryParameters; public static final fun parseForm (Ljava/lang/String;)Lorg/dexpace/kuri/query/QueryParameters; public final fun size ()I + public final fun split (Ljava/lang/String;C)Ljava/util/List; public final fun toFormUrlEncoded ()Ljava/lang/String; public final fun toMap ()Ljava/util/Map; public final fun toQueryString ()Ljava/lang/String; diff --git a/kuri/api/jvm/kuri.api b/kuri/api/jvm/kuri.api index 2f55010..134cccd 100644 --- a/kuri/api/jvm/kuri.api +++ b/kuri/api/jvm/kuri.api @@ -373,6 +373,34 @@ public final class org/dexpace/kuri/error/UriParseError$InvalidScheme : org/dexp public fun toString ()Ljava/lang/String; } +public final class org/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter : org/dexpace/kuri/error/UriParseError { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter; + public static synthetic fun copy$default (Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter;IIILjava/lang/Object;)Lorg/dexpace/kuri/error/UriParseError$IriBidiFormattingCharacter; + public fun equals (Ljava/lang/Object;)Z + public final fun getAt ()I + public final fun getCodePoint ()I + public fun getMessage ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class org/dexpace/kuri/error/UriParseError$IriInvalidCodePoint : org/dexpace/kuri/error/UriParseError { + public fun (II)V + public final fun component1 ()I + public final fun component2 ()I + public final fun copy (II)Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint; + public static synthetic fun copy$default (Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint;IIILjava/lang/Object;)Lorg/dexpace/kuri/error/UriParseError$IriInvalidCodePoint; + public fun equals (Ljava/lang/Object;)Z + public final fun getAt ()I + public final fun getCodePoint ()I + public fun getMessage ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class org/dexpace/kuri/error/UriParseError$MissingScheme : org/dexpace/kuri/error/UriParseError { public static final field INSTANCE Lorg/dexpace/kuri/error/UriParseError$MissingScheme; public fun equals (Ljava/lang/Object;)Z diff --git a/kuri/api/kuri.klib.api b/kuri/api/kuri.klib.api index d264fa1..c0a6ac7 100644 --- a/kuri/api/kuri.klib.api +++ b/kuri/api/kuri.klib.api @@ -160,6 +160,38 @@ sealed interface org.dexpace.kuri.error/UriParseError { // org.dexpace.kuri.erro final fun toString(): kotlin/String // org.dexpace.kuri.error/UriParseError.InvalidScheme.toString|toString(){}[0] } + final class IriBidiFormattingCharacter : org.dexpace.kuri.error/UriParseError { // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter|null[0] + constructor (kotlin/Int, kotlin/Int) // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.|(kotlin.Int;kotlin.Int){}[0] + + final val at // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.at|{}at[0] + final fun (): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.at.|(){}[0] + final val codePoint // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.codePoint|{}codePoint[0] + final fun (): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.codePoint.|(){}[0] + + final fun component1(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.component1|component1(){}[0] + final fun component2(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.component2|component2(){}[0] + final fun copy(kotlin/Int = ..., kotlin/Int = ...): org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.copy|copy(kotlin.Int;kotlin.Int){}[0] + final fun equals(kotlin/Any?): kotlin/Boolean // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.equals|equals(kotlin.Any?){}[0] + final fun hashCode(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.hashCode|hashCode(){}[0] + final fun toString(): kotlin/String // org.dexpace.kuri.error/UriParseError.IriBidiFormattingCharacter.toString|toString(){}[0] + } + + final class IriInvalidCodePoint : org.dexpace.kuri.error/UriParseError { // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint|null[0] + constructor (kotlin/Int, kotlin/Int) // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.|(kotlin.Int;kotlin.Int){}[0] + + final val at // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.at|{}at[0] + final fun (): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.at.|(){}[0] + final val codePoint // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.codePoint|{}codePoint[0] + final fun (): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.codePoint.|(){}[0] + + final fun component1(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.component1|component1(){}[0] + final fun component2(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.component2|component2(){}[0] + final fun copy(kotlin/Int = ..., kotlin/Int = ...): org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.copy|copy(kotlin.Int;kotlin.Int){}[0] + final fun equals(kotlin/Any?): kotlin/Boolean // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.equals|equals(kotlin.Any?){}[0] + final fun hashCode(): kotlin/Int // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.hashCode|hashCode(){}[0] + final fun toString(): kotlin/String // org.dexpace.kuri.error/UriParseError.IriInvalidCodePoint.toString|toString(){}[0] + } + final object EmptyHost : org.dexpace.kuri.error/UriParseError { // org.dexpace.kuri.error/UriParseError.EmptyHost|null[0] final fun equals(kotlin/Any?): kotlin/Boolean // org.dexpace.kuri.error/UriParseError.EmptyHost.equals|equals(kotlin.Any?){}[0] final fun hashCode(): kotlin/Int // org.dexpace.kuri.error/UriParseError.EmptyHost.hashCode|hashCode(){}[0] diff --git a/kuri/build.gradle.kts b/kuri/build.gradle.kts index c0ec576..32520cd 100644 --- a/kuri/build.gradle.kts +++ b/kuri/build.gradle.kts @@ -7,6 +7,8 @@ import com.vanniktech.maven.publish.JavadocJar import com.vanniktech.maven.publish.KotlinMultiplatform import com.vanniktech.maven.publish.SonatypeHost import kotlinx.kover.gradle.plugin.dsl.CoverageUnit +import kotlinx.validation.KotlinApiBuildTask +import kotlinx.validation.KotlinApiCompareTask import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.dsl.KotlinVersion @@ -160,6 +162,10 @@ kotlin { // on the Java test's compile classpath, at the version the Kotlin release was built for. jvmTest.dependencies { implementation(kotlin("test-junit")) + // Independent third-party UTS-46 oracle for IdnaIcu4jDifferentialTest (issue #66): + // icu4j has no multiplatform artifact, so it is scoped to the JVM test classpath only + // and must never be referenced from commonMain/commonTest. + implementation(libs.icu4j) } // Instrumented tests run under AndroidJUnitRunner (JUnit4), so bind kotlin.test to JUnit4 // and pull in the AndroidX test runtime that provides the runner. @@ -242,6 +248,72 @@ apiValidation { } } +// binary-compatibility-validator's built-in multiplatform auto-configuration only recognizes an +// Android target compilation literally named "release" (the classic AGP `com.android.library` + +// `androidTarget()` build-variant naming). The AGP KMP library plugin used here +// (`com.android.kotlin.multiplatform.library`) is single-variant and names its production +// compilation "main" instead, so that auto-configuration never matches it, `androidApiBuild`/ +// `androidApiCheck`/`androidApiDump` tasks never get created, and `apiCheck`/`apiDump` silently +// skip the android surface entirely (see the upstream gap tracked as KT-71172; verified locally +// by inspecting the plugin's target-matching source — it hasn't changed as of BCV 0.18.1). That +// silent skip is exactly how `kuri/api/android/kuri.api` went stale: nothing ever regenerated or +// checked it. Wire an equivalent android leg by hand using BCV's own public task classes, +// `KotlinApiBuildTask`/`KotlinApiCompareTask`, mirroring the classpath and directory layout +// (`api/android/kuri.api`) its Kotlin-target legs already use, and hook the result into the same +// `apiDump`/`apiCheck` entry points so android gets no special-cased invocation. +val androidAbiRuntimeClasspath: NamedDomainObjectProvider = + configurations.register("androidAbiRuntimeClasspath") { + description = "Runtime classpath for the manually wired android apiBuild/apiCheck tasks." + isCanBeConsumed = false + isCanBeResolved = true + isVisible = false + } + +dependencies.apply { + add(androidAbiRuntimeClasspath.name, libs.asm.core.get()) + add(androidAbiRuntimeClasspath.name, libs.asm.tree.get()) + add( + androidAbiRuntimeClasspath.name, + libs.kotlin.metadata.jvm + .get(), + ) +} + +val androidApiBuild: TaskProvider = + tasks.register("androidApiBuild") { + group = "verification" + description = "Builds the Kotlin API dump for the android target's 'main' compilation of kuri." + inputClassesDirs.from(tasks.named("compileAndroidMain").map { it.outputs.files }) + inputDependencies.from( + kotlin.targets + .getByName("android") + .compilations + .getByName("main") + .compileDependencyFiles, + ) + outputApiFile.set(layout.buildDirectory.file("kotlin/abi-android/kuri.api")) + runtimeClasspath.from(androidAbiRuntimeClasspath) + } + +val androidApiCheck: TaskProvider = + tasks.register("androidApiCheck") { + group = "verification" + description = "Checks the android target's public API against api/android/kuri.api." + projectApiFile.set(layout.projectDirectory.file("api/android/kuri.api")) + generatedApiFile.set(androidApiBuild.flatMap { it.outputApiFile }) + } + +val androidApiDump: TaskProvider = + tasks.register("androidApiDump") { + group = "other" + description = "Copies the android target's generated API dump into api/android/kuri.api." + from(androidApiBuild.flatMap { it.outputApiFile }) + into(layout.projectDirectory.dir("api/android")) + } + +tasks.named("apiCheck") { dependsOn(androidApiCheck) } +tasks.named("apiDump") { dependsOn(androidApiDump) } + // Publish every Kotlin Multiplatform target (plus the root `kotlinMultiplatform` publication) to Maven // Central through the Central Portal. The vanniktech plugin wires the per-target publications, a // Dokka-generated Javadoc jar and GPG signing; coordinates derive from the module `group` and the diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/Iri.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/Iri.kt index 173eb64..024f28c 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/Iri.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/Iri.kt @@ -20,10 +20,13 @@ import kotlin.jvm.JvmStatic * "no RFC 3986 deviations" invariant (Appendix B): `Uri.parse` still *rejects* raw non-ASCII, and * only this facility accepts it. * - * The mapping is a faithful, uniform §3.1 transform, not an IRI *validator*: the host is processed + * The mapping is a §3.1 transform layered on two RFC 3987 validation passes: the host is processed * by IDNA/UTS-46 (via the engine's ToASCII), and every other component has its non-ASCII code points - * percent-encoded as UTF-8 octets. No `ucschar`/`iprivate` repertoire is consulted, so a non-ASCII - * code point outside the IRI character set is encoded rather than rejected. + * percent-encoded as UTF-8 octets — but only once that code point has cleared §2.2's `ucschar`/ + * `iprivate` repertoire (`iprivate` legal only in the query) and §4.1's ban on the seven bidi + * formatting characters (LRM, RLM, LRE, RLE, PDF, LRO, RLO) anywhere in the IRI. §4.2's per-component + * directionality restriction is intentionally not enforced: the RFC states it as a SHOULD, not a + * MUST, and it is not relevant to every use of an IRI (e.g. one never presented visually). * * @see Uri */ @@ -37,9 +40,11 @@ public object Iri { * (never double-encoded). The fully-ASCII result is then validated and stored by the strict [Uri] * engine, so the returned [Uri] holds only RFC 3986-valid text. * - * The transform can fail: an IDNA-invalid non-ASCII host is a [ParseResult.Err], as is a mapping - * whose expanded ASCII form is rejected by the strict engine — note that percent-encoding grows - * the input, so a very long all-non-ASCII IRI may exceed the engine's maximum input length. + * The transform can fail: an IDNA-invalid non-ASCII host is a [ParseResult.Err], as is a non-ASCII + * code point outside its component's §2.2 repertoire, a §4.1 bidi formatting character anywhere + * in [iri], or a mapping whose expanded ASCII form is rejected by the strict engine — note that + * percent-encoding grows the input, so a very long all-non-ASCII IRI may exceed the engine's + * maximum input length. * * @param iri the internationalized reference to convert; non-ASCII input is accepted. * @return [ParseResult.Ok] with the mapped [Uri], or [ParseResult.Err] when the IRI cannot be diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/error/UriParseError.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/error/UriParseError.kt index 5550048..91ca69a 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/error/UriParseError.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/error/UriParseError.kt @@ -16,7 +16,8 @@ package org.dexpace.kuri.error * The parser produces a fixed slice of this catalog: structural failures * ([InvalidScheme], [MissingScheme], [InvalidPercentEncoding], [InvalidPort]), * host failures ([EmptyHost], [InvalidHost] carrying a [HostError], and - * [ForbiddenHostCodePoint]), and the size failure [InputTooLong]. The hierarchy + * [ForbiddenHostCodePoint]), the size failure [InputTooLong], and the RFC 3987 IRI-conversion + * failures ([IriInvalidCodePoint], [IriBidiFormattingCharacter]). The hierarchy * is `sealed` so a `when` over it is exhaustive without an `else`; * adding a variant is an intentional, API-visible change. */ @@ -39,6 +40,8 @@ public sealed interface UriParseError { is InvalidHost -> "invalid host \"$host\": $reason" is ForbiddenHostCodePoint -> "forbidden host code point $codePoint at offset $at" is InputTooLong -> "input length $length exceeds maximum $max" + is IriInvalidCodePoint -> "code point $codePoint at offset $at is outside the iri repertoire" + is IriBidiFormattingCharacter -> "forbidden bidi formatting character $codePoint at offset $at" } /** @@ -132,4 +135,31 @@ public sealed interface UriParseError { public val length: Int, public val max: Int, ) : UriParseError + + /** + * A non-ASCII code point in an IRI component falls outside the RFC 3987 §2.2 repertoire that + * component permits: outside `ucschar` for userinfo, path, and fragment; outside `ucschar` and + * `iprivate` for the query (the only component `iprivate` is legal in). The host is excluded from + * this check — it is independently validated by the IDNA/UTS-46 pipeline. + * + * @property codePoint the offending Unicode scalar value. + * @property at the offset of the offending code unit in the original input. + */ + public data class IriInvalidCodePoint( + public val codePoint: Int, + public val at: Int, + ) : UriParseError + + /** + * An IRI contains one of the seven bidirectional formatting characters (LRM, RLM, LRE, RLE, PDF, + * LRO, RLO) that RFC 3987 §4.1 forbids anywhere in an IRI's logical (stored/transmitted) form — + * they affect visual rendering only and are never themselves part of the IRI. + * + * @property codePoint the offending bidi formatting code point. + * @property at the offset of the offending code unit in the original input. + */ + public data class IriBidiFormattingCharacter( + public val codePoint: Int, + public val at: Int, + ) : UriParseError } diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt index 92965d1..44c8d23 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/idna/Idna.kt @@ -24,6 +24,18 @@ private const val ACE_PREFIX_LENGTH: Int = 4 /** Label separator inside a domain (U+002E FULL STOP); also the join separator. */ private const val LABEL_SEPARATOR: String = "." +/** [LABEL_SEPARATOR] as a `Char`, for splitting a raw (pre-mapping) domain by code point. */ +private const val LABEL_SEPARATOR_CHAR: Char = '.' + +/** U+3002 IDEOGRAPHIC FULL STOP: a UTS-46 dot-separator variant [IdnaMapping.Mapped] to `.`. */ +private const val IDEOGRAPHIC_FULL_STOP: Char = '\u3002' + +/** U+FF0E FULLWIDTH FULL STOP: a UTS-46 dot-separator variant [IdnaMapping.Mapped] to `.`. */ +private const val FULLWIDTH_FULL_STOP: Char = '\uFF0E' + +/** U+FF61 HALFWIDTH IDEOGRAPHIC FULL STOP: a UTS-46 dot-separator variant [IdnaMapping.Mapped] to `.`. */ +private const val HALFWIDTH_IDEOGRAPHIC_FULL_STOP: Char = '\uFF61' + /** * UTS-46 ToASCII / ToUnicode for IDNA domains (SPEC §7.4, [HOST-26]) under the `Url`-profile * parameter set ([HOST-28]): `CheckHyphens = false`, `CheckBidi = true`, `CheckJoiners = true`, @@ -91,29 +103,45 @@ internal object Idna { } /** - * Splits [domain] on [LABEL_SEPARATOR] and resolves each label independently: an all-ASCII label - * is lowercased and kept unconditionally (never validated, matching the existing intentional - * leniency — see [domainToAsciiForUrl]); a label carrying any non-ASCII code point runs the full - * [domainToAscii] pipeline, and that label's failure fails the whole domain. Splitting first and - * running NFC per label (inside [domainToAscii]) rather than over the whole domain is equivalent - * here: `U+002E` never participates in a cross-label canonical composition. + * Splits the raw (pre-mapping) [domain] on [LABEL_SEPARATOR] or any of the three non-ASCII + * UTS-46 dot-separator variants (see [splitRawLabels]) and resolves each raw label + * independently: a label that is already all-ASCII **before any mapping runs** is lowercased + * and kept unconditionally (never validated, matching the existing intentional leniency — see + * [domainToAsciiForUrl]); a label carrying any non-ASCII code point runs the full + * [domainToAscii] pipeline (which performs the mapping, NFC, and Punycode decode/validate steps + * itself), and that label's failure fails the whole domain. + * + * The ASCII/non-ASCII decision is deliberately made on each **raw** label, not on a + * whole-domain-mapped one: mapping the domain before this decision would let a non-ASCII + * sequence that maps down to a literal ASCII string (e.g. a fullwidth rendering of `xn--a`) + * take the unconditional-keep branch and skip [domainToAscii]'s Punycode decode/validate step + * entirely — exactly the case this leniency must not cover, since HOST-48 scopes it to a + * domain that is already ASCII pre-mapping. Splitting on the raw dot-separator variants (rather + * than only [LABEL_SEPARATOR]) keeps the boundary that the mapping step would otherwise reveal + * visible here too, without needing to map first: those three code points are always mapped to + * exactly [LABEL_SEPARATOR] on their own (never merged into a longer replacement), so a literal + * split on them is equivalent to mapping-then-splitting for boundary purposes alone. * * @return the joined ASCII domain, or `null` on the first non-ASCII label's UTS-46 failure. */ private fun asciiLenientDomainToAscii(domain: String): String? { - val labels = splitLabels(domain) + val labels = splitRawLabels(domain) val out = ArrayList(labels.size) - for (label in labels) { + var index = 0 + var failed = false + while (index < labels.size && !failed) { + val label = labels[index] if (label.isAllAscii()) { out.add(asciiLowercase(label)) } else { when (val result = domainToAscii(label)) { is ParseResult.Ok -> out.add(result.value) - is ParseResult.Err -> return null + is ParseResult.Err -> failed = true } } + index++ } - return out.joinToString(LABEL_SEPARATOR) + return if (failed) null else out.joinToString(LABEL_SEPARATOR) } /** @@ -173,6 +201,19 @@ internal object Idna { /** Splits [domain] into labels on [LABEL_SEPARATOR]; an empty domain yields one empty label. */ private fun splitLabels(domain: String): List = domain.split(LABEL_SEPARATOR) + /** + * Splits the raw, pre-mapping [domain] into labels on [LABEL_SEPARATOR] or any of the three + * non-ASCII UTS-46 dot-separator variants that map to it, so a label boundary they form is + * visible before mapping runs (see [asciiLenientDomainToAscii]). + */ + private fun splitRawLabels(domain: String): List = + domain.split( + LABEL_SEPARATOR_CHAR, + IDEOGRAPHIC_FULL_STOP, + FULLWIDTH_FULL_STOP, + HALFWIDTH_IDEOGRAPHIC_FULL_STOP, + ) + /** * Processes every label in order, short-circuiting on the first failure so the resulting * [ParseResult.Err] carries the original [domain] for diagnostics ([HOST-26] steps 4 & 5). diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriMapping.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriMapping.kt index a85614a..dab9f85 100644 --- a/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriMapping.kt +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriMapping.kt @@ -6,12 +6,16 @@ package org.dexpace.kuri.parser import org.dexpace.kuri.Uri import org.dexpace.kuri.error.ParseResult +import org.dexpace.kuri.error.UriParseError import org.dexpace.kuri.error.map import org.dexpace.kuri.idna.Idna import org.dexpace.kuri.percent.PercentCodec import org.dexpace.kuri.percent.PercentEncodeSets import org.dexpace.kuri.scheme.Scheme import org.dexpace.kuri.scheme.schemeColonIndex +import org.dexpace.kuri.text.NON_ASCII_MIN +import org.dexpace.kuri.text.charCount +import org.dexpace.kuri.text.codePointAt import org.dexpace.kuri.text.isAllAscii /** The authority-introducing `//` and the number of code units it spans (RFC 3986 §3.2). */ @@ -33,6 +37,13 @@ private const val DOUBLE_SLASH_LENGTH: Int = 2 * * Percent-encoding grows the input, so the engine's `MAX_INPUT_LENGTH` bound applies to the * *expanded* ASCII string; a very long all-non-ASCII IRI may exceed it after encoding and be rejected. + * + * Before any mapping happens, [precheckError] runs two RFC 3987 checks over the raw text: + * [findBidiFormattingCharacter] rejects a bidi formatting character (LRM, RLM, LRE, RLE, PDF, LRO, + * RLO) anywhere in [toUri]'s input (§4.1's MUST-NOT), and [repertoireError] rejects a non-ASCII code + * point outside `ucschar`/`iprivate` in userinfo, path, query, or fragment (§2.2's grammar) — + * `iprivate` is accepted only in the query, per the ABNF. The host is exempt from the repertoire + * check: it is independently validated by IDNA/UTS-46. */ @Suppress("TooManyFunctions") // One cohesive coarse splitter decomposed into single-purpose helpers, // each well under the 60-line cap; mirrors UriParser's structural decomposition. @@ -41,22 +52,92 @@ internal object IriMapping { * Maps [iri] to a [Uri] under RFC 3987 §3.1, or returns the first fatal failure. * * @param iri the internationalized reference to convert; non-ASCII input is accepted. - * @return [ParseResult.Ok] with the mapped [Uri], or [ParseResult.Err] on an IDNA host failure - * or a strict-engine rejection of the expanded ASCII form. + * @return [ParseResult.Ok] with the mapped [Uri], or [ParseResult.Err] on a §4.1 bidi-formatting + * violation, a §2.2 repertoire violation, an IDNA host failure, or a strict-engine rejection of + * the expanded ASCII form. */ internal fun toUri(iri: String): ParseResult { val parts = decompose(iri) check(parts.host == null || parts.hasAuthority) { "a host requires an authority" } - val mappedHost = - when (parts.host) { - null -> null - else -> - when (val host = mapHost(parts.host)) { - is ParseResult.Err -> return host - is ParseResult.Ok -> host.value - } + val hostResult = resolveHost(precheckError(iri, parts), parts) + return when (hostResult) { + is ParseResult.Err -> hostResult + is ParseResult.Ok -> UriParser.parse(reassemble(parts, hostResult.value)).map { Uri(it) } + } + } + + /** + * The first §4.1 bidi-formatting or §2.2 repertoire violation in [iri], or `null` when both checks + * pass. Runs before any mapping so a rejected IRI never reaches IDNA or percent-encoding. + */ + private fun precheckError( + iri: String, + parts: IriParts, + ): UriParseError? = findBidiFormattingCharacter(iri) ?: repertoireError(parts) + + /** + * Resolves [parts]'s host to its mapped ASCII form, short-circuiting to [precheck] first: a + * precheck failure (bidi or repertoire) must win over a host failure so the earliest structural + * problem in [iri]'s own text is what gets reported, not one downstream of it. `null` means "no + * host" (a hostless reference), distinct from a mapping failure. + */ + private fun resolveHost( + precheck: UriParseError?, + parts: IriParts, + ): ParseResult = + when { + precheck != null -> ParseResult.Err(precheck) + parts.host == null -> ParseResult.Ok(null) + else -> mapHost(parts.host) + } + + /** + * The first of RFC 3987 §4.1's seven forbidden bidi formatting characters in [iri], or `null`. + * The rule is IRI-wide, not scoped to a single component, so this scans the raw, undecomposed text. + */ + private fun findBidiFormattingCharacter(iri: String): UriParseError? { + for (index in iri.indices) { + val codePoint = iri[index].code + if (IriRepertoire.isBidiFormattingCharacter(codePoint)) { + return UriParseError.IriBidiFormattingCharacter(codePoint, index) } - return UriParser.parse(reassemble(parts, mappedHost)).map { Uri(it) } + } + return null + } + + /** + * The first raw component (userinfo, path, query, then fragment) that carries a non-ASCII code + * point outside its RFC 3987 §2.2 repertoire, or `null`. The host is exempt: it is validated + * separately by the IDNA/UTS-46 pipeline in [mapHost]. + */ + private fun repertoireError(parts: IriParts): UriParseError? = + parts.userInfo?.let { validateRepertoire(it, allowIprivate = false) } + ?: validateRepertoire(parts.path, allowIprivate = false) + ?: parts.query?.let { validateRepertoire(it, allowIprivate = true) } + ?: parts.fragment?.let { validateRepertoire(it, allowIprivate = false) } + + /** + * Scans [located]'s text code point by code point, failing on the first non-ASCII value outside + * `ucschar` (and, when [allowIprivate], also outside `iprivate`) — see RFC 3987 §2.2. [Located.offset] + * is the component's start in the original IRI, so the reported failure locates the exact input index. + */ + private fun validateRepertoire( + located: Located, + allowIprivate: Boolean, + ): UriParseError? { + var index = 0 + while (index < located.text.length) { + val codePoint = codePointAt(located.text, index) + val legal = + codePoint < NON_ASCII_MIN || + IriRepertoire.isUcschar(codePoint) || + (allowIprivate && IriRepertoire.isIprivate(codePoint)) + if (!legal) { + return UriParseError.IriInvalidCodePoint(codePoint, located.offset + index) + } + index += charCount(codePoint) + } + return null } /** @@ -80,9 +161,9 @@ internal object IriMapping { val out = StringBuilder() parts.scheme?.let { out.append(it).append(':') } if (parts.hasAuthority) appendAuthority(out, parts, mappedHost) - out.append(encodeComponent(parts.path)) - parts.query?.let { out.append('?').append(encodeComponent(it)) } - parts.fragment?.let { out.append('#').append(encodeComponent(it)) } + out.append(encodeComponent(parts.path.text)) + parts.query?.let { out.append('?').append(encodeComponent(it.text)) } + parts.fragment?.let { out.append('#').append(encodeComponent(it.text)) } return out.toString() } @@ -95,7 +176,7 @@ internal object IriMapping { require(parts.hasAuthority) { "authority reassembly requires an authority" } val host = requireNotNull(mappedHost) { "a present authority requires a mapped host" } out.append(DOUBLE_SLASH) - parts.userInfo?.let { out.append(encodeComponent(it)).append('@') } + parts.userInfo?.let { out.append(encodeComponent(it.text)).append('@') } out.append(host) parts.port?.let { out.append(':').append(it) } } @@ -110,15 +191,16 @@ internal object IriMapping { /** Splits [iri] at its ASCII structural delimiters into the components the transform maps. */ private fun decompose(iri: String): IriParts { val hash = iri.indexOf('#') - val fragment = if (hash < 0) null else iri.substring(hash + 1) + val fragment = if (hash < 0) null else Located(iri.substring(hash + 1), hash + 1) val body = if (hash < 0) iri else iri.substring(0, hash) val mark = body.indexOf('?') - val query = if (mark < 0) null else body.substring(mark + 1) + val query = if (mark < 0) null else Located(body.substring(mark + 1), mark + 1) val hier = if (mark < 0) body else body.substring(0, mark) val scheme = detectScheme(hier) check(scheme == null || hier.length > scheme.length) { "a scheme requires a trailing colon" } + val restOffset = if (scheme == null) 0 else scheme.length + 1 val rest = if (scheme == null) hier else hier.substring(scheme.length + 1) - return splitAuthorityPath(scheme, rest, query, fragment) + return splitAuthorityPath(scheme, Located(rest, restOffset), query, fragment) } /** A scheme is the prefix before a `:` that precedes any `/`, and only when it is a valid scheme. */ @@ -132,17 +214,23 @@ internal object IriMapping { /** Routes [rest] to an authority split when it opens with `//`, else treats all of it as the path. */ private fun splitAuthorityPath( scheme: String?, - rest: String, - query: String?, - fragment: String?, + rest: Located, + query: Located?, + fragment: Located?, ): IriParts { - if (!rest.startsWith(DOUBLE_SLASH)) { + if (!rest.text.startsWith(DOUBLE_SLASH)) { return IriParts(scheme, hasAuthority = false, path = rest, query = query, fragment = fragment) } - val slash = rest.indexOf('/', DOUBLE_SLASH_LENGTH) - val end = if (slash < 0) rest.length else slash - val authority = splitAuthority(rest.substring(DOUBLE_SLASH_LENGTH, end)) - val path = if (slash < 0) "" else rest.substring(slash) + val slash = rest.text.indexOf('/', DOUBLE_SLASH_LENGTH) + val end = if (slash < 0) rest.text.length else slash + val authorityText = rest.text.substring(DOUBLE_SLASH_LENGTH, end) + val authority = splitAuthority(Located(authorityText, rest.offset + DOUBLE_SLASH_LENGTH)) + val path = + if (slash < 0) { + Located("", rest.offset + rest.text.length) + } else { + Located(rest.text.substring(slash), rest.offset + slash) + } return IriParts( scheme = scheme, hasAuthority = true, @@ -156,16 +244,16 @@ internal object IriMapping { } /** Splits authority text into userinfo (before the LAST `@`) and host[:port]. */ - private fun splitAuthority(authority: String): Authority { - val at = authority.lastIndexOf('@') - val userInfo = if (at < 0) null else authority.substring(0, at) - val hostPort = if (at < 0) authority else authority.substring(at + 1) + private fun splitAuthority(authority: Located): Authority { + val at = authority.text.lastIndexOf('@') + val userInfo = if (at < 0) null else Located(authority.text.substring(0, at), authority.offset) + val hostPort = if (at < 0) authority.text else authority.text.substring(at + 1) return if (hostPort.startsWith('[')) splitBracketed(userInfo, hostPort) else splitPlain(userInfo, hostPort) } /** A non-bracketed host ends at the first `:`, whose tail is the port. */ private fun splitPlain( - userInfo: String?, + userInfo: Located?, hostPort: String, ): Authority { val colon = hostPort.indexOf(':') @@ -177,7 +265,7 @@ internal object IriMapping { /** A bracketed literal's port colon, if any, immediately follows the closing `]`. */ private fun splitBracketed( - userInfo: String?, + userInfo: Located?, hostPort: String, ): Authority { val close = hostPort.indexOf(']') @@ -188,21 +276,30 @@ internal object IriMapping { return Authority(userInfo, host, port) } + /** + * A raw substring of the original IRI together with its start offset, so a later validation pass + * (the RFC 3987 §2.2 repertoire check) can report the exact input index of a rejected code point. + */ + private data class Located( + val text: String, + val offset: Int, + ) + /** The located structural components of an IRI, each carried raw (pre-transform). */ private data class IriParts( val scheme: String?, val hasAuthority: Boolean, - val userInfo: String? = null, + val userInfo: Located? = null, val host: String? = null, val port: String? = null, - val path: String, - val query: String?, - val fragment: String?, + val path: Located, + val query: Located? = null, + val fragment: Located? = null, ) /** The located authority sub-components: raw userinfo (or `null`), raw host, and raw port (or `null`). */ private data class Authority( - val userInfo: String?, + val userInfo: Located?, val host: String, val port: String?, ) diff --git a/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriRepertoire.kt b/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriRepertoire.kt new file mode 100644 index 0000000..d8c63ba --- /dev/null +++ b/kuri/src/commonMain/kotlin/org/dexpace/kuri/parser/IriRepertoire.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2026 dexpace and Omar Aljarrah + * SPDX-License-Identifier: MIT + */ +package org.dexpace.kuri.parser + +/** Separates the inclusive `-` records within one of this file's hex range blobs. */ +private const val RECORD_SEPARATOR: Char = ';' + +/** Separates the start and end hex bounds of one inclusive range record. */ +private const val RANGE_SEPARATOR: Char = '-' + +/** Radix the hex bounds and bidi-formatting code points are parsed in. */ +private const val RADIX_HEX: Int = 16 + +/** + * The RFC 3987 §2.2 `ucschar`/`iprivate` repertoire and the §4.1 bidi-formatting-character set. + * + * Both tables are fixed by the RFC's own ABNF (not tied to any Unicode version), so unlike the + * generated IDNA/UTS-46 tables they never need regeneration. They are still stored as parsed hex + * text rather than `IntRange` literals, mirroring the codegen'd IDNA range tables: it keeps every + * bound traceable one-for-one to the ABNF quoted below without a wall of unnamed numeric literals. + */ +internal object IriRepertoire { + /** + * The `ucschar` ranges (RFC 3987 §2.2): + * `%xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD / + * %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD / %x70000-7FFFD / %x80000-8FFFD / + * %x90000-9FFFD / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD / %xD0000-DFFFD / + * %xE1000-EFFFD`. These are the code points [iunreserved] may hold beyond + * `ALPHA`/`DIGIT`/`"-"`/`"."`/`"_"`/`"~"`, legal in every non-host component this facility maps. + */ + private val ucscharRanges: List = + decodeRanges( + "A0-D7FF;F900-FDCF;FDF0-FFEF;10000-1FFFD;20000-2FFFD;30000-3FFFD;40000-4FFFD;" + + "50000-5FFFD;60000-6FFFD;70000-7FFFD;80000-8FFFD;90000-9FFFD;A0000-AFFFD;" + + "B0000-BFFFD;C0000-CFFFD;D0000-DFFFD;E1000-EFFFD", + ) + + /** + * The `iprivate` ranges (RFC 3987 §2.2): `%xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD`. Per + * the ABNF, `iprivate` appears only in `iquery` (`iquery = *( ipchar / iprivate / "/" / "?" )`) — + * it is not part of `ipchar`, so it is illegal in userinfo, path, and fragment even though it is + * legal in the query. + */ + private val iprivateRanges: List = decodeRanges("E000-F8FF;F0000-FFFFD;100000-10FFFD") + + /** + * The seven bidirectional formatting characters RFC 3987 §4.1 forbids anywhere in an IRI: LRM + * (U+200E), RLM (U+200F), LRE (U+202A), RLE (U+202B), PDF (U+202C), LRO (U+202D), RLO (U+202E). + * They affect visual rendering only and must never appear in the logical (stored/transmitted) form. + */ + private val bidiFormattingCharacters: Set = + "200E,200F,202A,202B,202C,202D,202E".split(',').map { it.toInt(RADIX_HEX) }.toSet() + + /** + * True when [codePoint] is in one of the `ucschar` ranges (RFC 3987 §2.2). + * + * @param codePoint the Unicode scalar value to test. + * @return `true` iff [codePoint] falls within a `ucschar` range. + */ + internal fun isUcschar(codePoint: Int): Boolean = ucscharRanges.any { codePoint in it } + + /** + * True when [codePoint] is in one of the `iprivate` ranges (RFC 3987 §2.2), legal only in the + * query component. + * + * @param codePoint the Unicode scalar value to test. + * @return `true` iff [codePoint] falls within an `iprivate` range. + */ + internal fun isIprivate(codePoint: Int): Boolean = iprivateRanges.any { codePoint in it } + + /** + * True when [codePoint] is one of the seven bidi formatting characters RFC 3987 §4.1 forbids. + * + * @param codePoint the Unicode scalar value to test. + * @return `true` iff [codePoint] is LRM, RLM, LRE, RLE, PDF, LRO, or RLO. + */ + internal fun isBidiFormattingCharacter(codePoint: Int): Boolean = codePoint in bidiFormattingCharacters + + /** Decodes a semicolon-joined `-` hex blob into its inclusive [IntRange]s. */ + private fun decodeRanges(blob: String): List = + blob.split(RECORD_SEPARATOR).map { record -> + val dash = record.indexOf(RANGE_SEPARATOR) + check(dash > 0) { "malformed range record: $record" } + record.substring(0, dash).toInt(RADIX_HEX)..record.substring(dash + 1).toInt(RADIX_HEX) + } +} diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt index 3347126..f3c48e7 100644 --- a/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/IriTest.kt @@ -101,13 +101,101 @@ class IriTest { } @Test - fun `encodes any non-ascii code point even outside the iri repertoire`() { - // U+FDD0 is a noncharacter: neither ucschar nor iprivate, yet the converter still encodes it. + fun `rejects a non-ascii code point outside the iri repertoire`() { + // U+FDD0 is a noncharacter: neither ucschar nor iprivate, so RFC 3987 §2.2 excludes it. val nonCharacter = Char(0xFDD0).toString() + val iri = "http://h/$nonCharacter" - val uri = Iri.toUri("http://h/$nonCharacter").getOrThrow() + val error = assertIs(assertIs(Iri.toUri(iri)).error) - assertEquals("/%EF%B7%90", uri.encodedPath) + assertEquals(0xFDD0, error.codePoint) + assertEquals(iri.indexOf(nonCharacter), error.at) + } + + @Test + fun `accepts an iprivate code point only in the query component`() { + // U+E000 is iprivate: legal in iquery (RFC 3987 §2.2) but not in path, userinfo, or fragment. + val iprivate = Char(0xE000).toString() + + val uri = Iri.toUri("http://h/?q=$iprivate").getOrThrow() + + assertEquals("q=%EE%80%80", uri.query) + } + + @Test + fun `rejects an iprivate code point in the path`() { + val iprivate = Char(0xE000).toString() + val iri = "http://h/$iprivate" + + val error = assertIs(assertIs(Iri.toUri(iri)).error) + + assertEquals(0xE000, error.codePoint) + assertEquals(iri.indexOf(iprivate), error.at) + } + + @Test + fun `rejects an iprivate code point in userinfo`() { + val iprivate = Char(0xE000).toString() + val iri = "http://u$iprivate@h/" + + assertIs(assertIs(Iri.toUri(iri)).error) + } + + @Test + fun `rejects an iprivate code point in the fragment`() { + val iprivate = Char(0xE000).toString() + val iri = "http://h/#$iprivate" + + assertIs(assertIs(Iri.toUri(iri)).error) + } + + @Test + fun `rejects a bidi formatting character anywhere in the iri`() { + // U+200E LEFT-TO-RIGHT MARK is one of the seven formatting characters RFC 3987 §4.1 forbids. + val lrm = Char(0x200E).toString() + val iri = "http://h/$lrm" + + val error = + assertIs(assertIs(Iri.toUri(iri)).error) + + assertEquals(0x200E, error.codePoint) + assertEquals(iri.indexOf(lrm), error.at) + } + + @Test + fun `a bidi violation anywhere in the iri wins over an earlier repertoire violation`() { + // U+FDD0 is a repertoire violation (neither ucschar nor iprivate, see the noncharacter test + // above) placed BEFORE the LRM in raw text order. A naive single combined left-to-right scan + // would report the noncharacter first; the bidi scan is instead its own complete first pass + // over the whole raw string, so the later LRM wins the precedence, not just a leftmost match. + val nonCharacter = Char(0xFDD0).toString() + val lrm = Char(0x200E).toString() + val iri = "http://h/$nonCharacter$lrm" + + val error = + assertIs(assertIs(Iri.toUri(iri)).error) + + assertEquals(0x200E, error.codePoint) + assertEquals(iri.indexOf(lrm), error.at) + } + + @Test + fun `a precheck failure wins over an idna host failure`() { + // The leading combining mark makes this host idna-invalid on its own — see + // `fails when a non-ascii host is idna-invalid` above, which proves the same host text + // fails with UriParseError.InvalidHost(HostError.IdnaFailed) when used alone. Adding a + // repertoire-violating noncharacter to the path must still surface the precheck failure + // instead, since resolveHost short-circuits to the precheck error before ever mapping the + // host (see its KDoc). + val leadingCombiningMark = Char(0x0301).toString() + val nonCharacter = Char(0xFDD0).toString() + val invalidHost = "${leadingCombiningMark}x" + val iri = "http://$invalidHost/$nonCharacter" + + val error = assertIs(assertIs(Iri.toUri(iri)).error) + + assertEquals(0xFDD0, error.codePoint) + assertEquals(iri.indexOf(nonCharacter), error.at) } @Test diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt index 0894067..7bb4377 100644 --- a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnTest.kt @@ -9,6 +9,7 @@ import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertNotNull import kotlin.test.assertNull +import kotlin.test.assertTrue /** * Behavioural tests for the public [Idn] facade over the internal [Idna] engine (UTS-46, SPEC §7.4). @@ -45,6 +46,54 @@ class IdnTest { assertEquals("xn--a.xn--bcher-kva", Idn.toAscii(input).getOrNull()) } + @Test + fun `toAscii recognizes the ideographic full stop as a per-label leniency separator`() { + // U+3002 IDEOGRAPHIC FULL STOP is one of the three non-ASCII UTS-46 dot-separator + // variants mapped to U+002E; the per-label ASCII/non-ASCII gate must see the boundary it + // forms, not just a literal '.', so "xn--a" still gets the malformed-Punycode leniency. + val ideographicFullStop = Char(0x3002).toString() + val input = "xn--a" + ideographicFullStop + "b" + uUmlaut + "cher" + val expected = Idn.toAscii("xn--a.b" + uUmlaut + "cher") + + assertTrue(expected.isOk()) + assertEquals(expected.getOrNull(), Idn.toAscii(input).getOrNull()) + } + + @Test + fun `toAscii recognizes the fullwidth full stop as a per-label leniency separator`() { + // U+FF0E FULLWIDTH FULL STOP is another UTS-46 dot-separator variant mapped to U+002E. + val fullwidthFullStop = Char(0xFF0E).toString() + val input = "xn--a" + fullwidthFullStop + "b" + uUmlaut + "cher" + val expected = Idn.toAscii("xn--a.b" + uUmlaut + "cher") + + assertTrue(expected.isOk()) + assertEquals(expected.getOrNull(), Idn.toAscii(input).getOrNull()) + } + + @Test + fun `toAscii recognizes the halfwidth ideographic full stop as a per-label leniency separator`() { + // U+FF61 HALFWIDTH IDEOGRAPHIC FULL STOP is the third UTS-46 dot-separator variant mapped + // to U+002E. + val halfwidthIdeographicFullStop = Char(0xFF61).toString() + val input = "xn--a" + halfwidthIdeographicFullStop + "b" + uUmlaut + "cher" + val expected = Idn.toAscii("xn--a.b" + uUmlaut + "cher") + + assertTrue(expected.isOk()) + assertEquals(expected.getOrNull(), Idn.toAscii(input).getOrNull()) + } + + @Test + fun `toAscii rejects a fullwidth label that maps down to a literal xn-- ascii string`() { + // U+FF58 U+FF4E U+FF0D U+FF0D 'a' (fullwidth x, n, hyphen-minus x2, then ascii 'a') maps + // down to the literal ASCII text "xn--a" under UTS-46 mapping. The per-label ASCII/ + // non-ASCII routing decision must be made on this original non-ASCII label, not on its + // post-mapping text, or the label slips past the Punycode decode/validate step that + // Idna.domainToAscii("xn--a") itself fails. + val fullwidthLabel = Char(0xFF58).toString() + Char(0xFF4E) + Char(0xFF0D) + Char(0xFF0D) + "a" + + assertFalse(Idn.toAscii(fullwidthLabel).isOk()) + } + @Test fun `toAscii fails on a disallowed code point`() { // U+0080 is a plain disallowed code point, so ToASCII must reject it fatally. diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt index 63902e4..97c5b47 100644 --- a/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/idna/IdnaConformanceTest.kt @@ -31,6 +31,15 @@ import kotlin.test.assertTrue * by [Idna.domainToAscii] (so the corpus's required rejection is a host obligation, not a ToASCII one) * yet rejected by [UrlHostParser] under the `Url` profile. The suite also ratchets: it fails if any other * case regresses, or if the residual drifts from the live failing set. + * + * WPT is the independent oracle this suite checks kuri against, not `tools/internal/idnaref` (the Go + * reference the codegen generator uses to derive [IDNA_KNOWN_FAILURES]). `idnaref` is a faithful Go port + * of this same [Idna.domainToAscii] algorithm — a second implementation, not a second spec reading — so a + * conceptual bug shared between the Kotlin engine and its port would reproduce identically in both and + * pass the derivation undetected; its value is catching a porting slip between the two, not verifying the + * algorithm against an outside source. That verification is what running every case here directly against + * the WPT corpus itself provides. `IdnaIcu4jDifferentialTest` (JVM-only) adds a further, genuinely + * third-party cross-check against ICU4J's UTS-46 implementation over the same corpus. */ class IdnaConformanceTest { private val residual: Set = IDNA_KNOWN_FAILURES diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedConformanceTest.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedConformanceTest.kt new file mode 100644 index 0000000..e433c68 --- /dev/null +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedConformanceTest.kt @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2026 dexpace and Omar Aljarrah + * SPDX-License-Identifier: MIT + */ + +package org.dexpace.kuri.query + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Runs every WPT `urlencoded-parser.json` case (modelled as [URL_ENCODED_TEST_CASES]) through + * [FormUrlEncoded.parse] and ratchets a tracked known-failures baseline (SPEC §10.4, [QUERY-21]). + * + * The corpus is the array literal WPT's `url/urlencoded-parser.any.js` feeds to its + * "URLSearchParams constructed with" subtest: a flat `{input, output: [[name, value], ...]}` + * shape with no `failure` field, so [FormUrlEncoded.parse]'s return type + * (`List>`) already matches [UrlEncodedCase.output] with no adaptation. The + * file's two sibling per-entry subtests (`request.formData()`/`response.formData()`) exercise + * non-UTF-8 charsets (`windows-1252`, `shift_jis`) and are out of scope, since kuri's form codec + * is UTF-8-only; they are not vendored. + * + * There is no residual divergence: [KNOWN_FAILURES] is empty and the suite is at full + * conformance. It still ratchets in both directions -- a brand-new failure breaks the + * untracked-regressions test, and a regression that repopulates the live set breaks the + * baseline-equality test until the baseline is updated. + */ +class UrlEncodedConformanceTest { + /** True when parsing [case]'s input reproduces its expected output pairs, in order. */ + private fun caseMatches(case: UrlEncodedCase): Boolean = FormUrlEncoded.parse(case.input) == case.output + + /** The inputs of every case the live parser currently does not satisfy. */ + private fun liveFailingInputs(): Set = + URL_ENCODED_TEST_CASES.filterNot { caseMatches(it) }.map { it.input }.toSet() + + @Test + fun `every urlencoded case outside the known-failures set matches WPT`() { + val untracked = liveFailingInputs() - KNOWN_FAILURES + assertTrue(untracked.isEmpty(), "new urlencoded-parser regressions (untracked failures): $untracked") + } + + @Test + fun `the known-failures set exactly equals the live failing set`() { + // Ratchet: a fixed gap (tracked failure now passing) or a brand-new failure breaks this + // until the baseline is regenerated, so the deferred debt can never drift silently. + assertEquals(KNOWN_FAILURES, liveFailingInputs()) + } + + @Test + fun `every tracked known failure is a real corpus case`() { + val inputs = URL_ENCODED_TEST_CASES.map { it.input }.toSet() + val orphans = KNOWN_FAILURES - inputs + assertTrue(orphans.isEmpty(), "known failures absent from the corpus: $orphans") + } + + @Test + fun `the corpus is non-empty and fully conformant`() { + assertTrue(URL_ENCODED_TEST_CASES.isNotEmpty(), "the WPT corpus should not be empty") + val failing = liveFailingInputs() + assertTrue(failing.isEmpty(), "every WPT case must parse per spec; failing: $failing") + } + + private companion object { + /** + * The tracked baseline of currently-failing case inputs. It is empty: the live form parser + * satisfies every WPT `urlencoded-parser.json` case. The ratcheting + * `the known-failures set exactly equals the live failing set` test pins the suite at full + * conformance: any regression repopulates the live set and breaks the build until this + * baseline is updated. + */ + private val KNOWN_FAILURES: Set = emptySet() + } +} diff --git a/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedTestData.kt b/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedTestData.kt new file mode 100644 index 0000000..c5b0cce --- /dev/null +++ b/kuri/src/commonTest/kotlin/org/dexpace/kuri/query/UrlEncodedTestData.kt @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2026 dexpace and Omar Aljarrah + * SPDX-License-Identifier: MIT + */ + +// Generated bulk data, not hand-written logic: the chunked builders intentionally exceed +// detekt's method/class-size heuristics to stay within the 64 KB JVM method limit. +@file:Suppress("LongMethod", "LargeClass", "MatchingDeclarationName") + +package org.dexpace.kuri.query + +/** + * One WPT `urlencoded-parser.json` case: parsing [input] as `application/x-www-form-urlencoded` + * must yield [output], the ordered list of decoded `(name, value)` pairs. + */ +internal data class UrlEncodedCase( + val input: String, + val output: List>, +) + +// Generated by ./gradlew generateUrlEncodedTestData from the WPT urlencoded-parser corpus. +// Chunked into small builders so no single method exceeds the 64 KB JVM limit. +private object UrlEncodedCaseData { + fun all(): List = part0() + + private fun part0(): List = + listOf( + UrlEncodedCase( + input = "test", + output = + listOf( + "test" to "", + ), + ), + UrlEncodedCase( + input = "\ufefftest=\ufeff", + output = + listOf( + "\ufefftest" to "\ufeff", + ), + ), + UrlEncodedCase( + input = "%EF%BB%BFtest=%EF%BB%BF", + output = + listOf( + "\ufefftest" to "\ufeff", + ), + ), + UrlEncodedCase( + input = "%EF%BF%BF=%EF%BF%BF", + output = + listOf( + "\uffff" to "\uffff", + ), + ), + UrlEncodedCase( + input = "%FE%FF", + output = + listOf( + "\ufffd\ufffd" to "", + ), + ), + UrlEncodedCase( + input = "%FF%FE", + output = + listOf( + "\ufffd\ufffd" to "", + ), + ), + UrlEncodedCase( + input = "\u2020&\u2020=x", + output = + listOf( + "\u2020" to "", + "\u2020" to "x", + ), + ), + UrlEncodedCase( + input = "%C2", + output = + listOf( + "\ufffd" to "", + ), + ), + UrlEncodedCase( + input = "%C2x", + output = + listOf( + "\ufffdx" to "", + ), + ), + UrlEncodedCase( + input = "_charset_=windows-1252&test=%C2x", + output = + listOf( + "_charset_" to "windows-1252", + "test" to "\ufffdx", + ), + ), + UrlEncodedCase( + input = "", + output = emptyList(), + ), + UrlEncodedCase( + input = "a", + output = + listOf( + "a" to "", + ), + ), + UrlEncodedCase( + input = "a=b", + output = + listOf( + "a" to "b", + ), + ), + UrlEncodedCase( + input = "a=", + output = + listOf( + "a" to "", + ), + ), + UrlEncodedCase( + input = "=b", + output = + listOf( + "" to "b", + ), + ), + UrlEncodedCase( + input = "&", + output = emptyList(), + ), + UrlEncodedCase( + input = "&a", + output = + listOf( + "a" to "", + ), + ), + UrlEncodedCase( + input = "a&", + output = + listOf( + "a" to "", + ), + ), + UrlEncodedCase( + input = "a&a", + output = + listOf( + "a" to "", + "a" to "", + ), + ), + UrlEncodedCase( + input = "a&b&c", + output = + listOf( + "a" to "", + "b" to "", + "c" to "", + ), + ), + UrlEncodedCase( + input = "a=b&c=d", + output = + listOf( + "a" to "b", + "c" to "d", + ), + ), + UrlEncodedCase( + input = "a=b&c=d&", + output = + listOf( + "a" to "b", + "c" to "d", + ), + ), + UrlEncodedCase( + input = "&&&a=b&&&&c=d&", + output = + listOf( + "a" to "b", + "c" to "d", + ), + ), + UrlEncodedCase( + input = "a=a&a=b&a=c", + output = + listOf( + "a" to "a", + "a" to "b", + "a" to "c", + ), + ), + UrlEncodedCase( + input = "a==a", + output = + listOf( + "a" to "=a", + ), + ), + UrlEncodedCase( + input = "a=a+b+c+d", + output = + listOf( + "a" to "a b c d", + ), + ), + UrlEncodedCase( + input = "%=a", + output = + listOf( + "%" to "a", + ), + ), + UrlEncodedCase( + input = "%a=a", + output = + listOf( + "%a" to "a", + ), + ), + UrlEncodedCase( + input = "%a_=a", + output = + listOf( + "%a_" to "a", + ), + ), + UrlEncodedCase( + input = "%61=a", + output = + listOf( + "a" to "a", + ), + ), + UrlEncodedCase( + input = "%61+%4d%4D=", + output = + listOf( + "a MM" to "", + ), + ), + UrlEncodedCase( + input = "id=0&value=%", + output = + listOf( + "id" to "0", + "value" to "%", + ), + ), + UrlEncodedCase( + input = "b=%2sf%2a", + output = + listOf( + "b" to "%2sf*", + ), + ), + UrlEncodedCase( + input = "b=%2%2af%2a", + output = + listOf( + "b" to "%2*f*", + ), + ), + UrlEncodedCase( + input = "b=%%2a", + output = + listOf( + "b" to "%*", + ), + ), + ) +} + +/** Every WPT `urlencoded-parser.json` case (objects carrying an `input`). */ +internal val URL_ENCODED_TEST_CASES: List = UrlEncodedCaseData.all() diff --git a/kuri/src/jvmTest/kotlin/org/dexpace/kuri/idna/IdnaIcu4jDifferentialTest.kt b/kuri/src/jvmTest/kotlin/org/dexpace/kuri/idna/IdnaIcu4jDifferentialTest.kt new file mode 100644 index 0000000..fe968c7 --- /dev/null +++ b/kuri/src/jvmTest/kotlin/org/dexpace/kuri/idna/IdnaIcu4jDifferentialTest.kt @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2026 dexpace and Omar Aljarrah + * SPDX-License-Identifier: MIT + */ +package org.dexpace.kuri.idna + +import com.ibm.icu.text.IDNA +import org.dexpace.kuri.error.ParseResult +import kotlin.test.Test +import kotlin.test.assertTrue + +/** + * Cross-checks [Idna.domainToAscii] against ICU4J's UTS-46 implementation over the WPT IDNA corpus + * ([IDNA_CONFORMANCE_CASES]) — a genuinely third-party oracle, unlike `tools/internal/idnaref` (see + * [IdnaConformanceTest]), which only re-derives kuri's own algorithm as a second implementation. + * + * kuri and ICU4J legitimately disagree in two structural, documented ways rather than everywhere they + * differ being swallowed as noise: + * - ICU4J's UTS46 implementation has no switch to disable `CheckHyphens` or `VerifyDnsLength`: it always + * rejects a leading/trailing hyphen, a "??--" third/fourth-position hyphen, an empty label, or a + * length overflow ([structuralOptionErrors]). kuri's `Url` profile deliberately sets both to `false` + * (WHATWG requires the lenient defaults for URL hosts), so a case built to exercise one of those + * relaxations disagrees by construction, not by defect. + * - ICU4J 77.1 bundles Unicode 16.0 data; kuri bundles Unicode 17.0. A handful of corpus inputs carry + * CJK ideographs newly assigned in 17.0 that ICU4J's older tables still treat as unassigned + * ([unicodeVersionSkewInputs]), pinned by exact input and ratcheted so a Unicode-version change on + * either side forces a conscious update instead of a silent drift. + * + * Every other disagreement fails the suite: nothing outside those two documented causes is swallowed. + */ +class IdnaIcu4jDifferentialTest { + private val icu: IDNA = IDNA.getUTS46Instance(IDNA.DEFAULT or IDNA.CHECK_BIDI or IDNA.CHECK_CONTEXTJ) + + /** + * ICU4J [IDNA.Error] categories ICU4J's UTS46 implementation always enforces with no equivalent of + * kuri's `Url`-profile `CheckHyphens = false` / `VerifyDnsLength = false` relaxations, each mapped to + * the kuri profile rule that explains a case flagged with it. + */ + private val structuralOptionErrors: Map = + mapOf( + IDNA.Error.LEADING_HYPHEN to "CheckHyphens=false permits a leading hyphen; ICU4J has no such switch", + IDNA.Error.TRAILING_HYPHEN to "CheckHyphens=false permits a trailing hyphen; ICU4J has no such switch", + IDNA.Error.HYPHEN_3_4 to "CheckHyphens=false permits a 3rd/4th-position hyphen; ICU4J has no such switch", + IDNA.Error.EMPTY_LABEL to "VerifyDnsLength=false permits an empty label; ICU4J always rejects one", + IDNA.Error.LABEL_TOO_LONG to "VerifyDnsLength=false skips the 63-octet label cap ICU4J always enforces", + IDNA.Error.DOMAIN_NAME_TOO_LONG to "VerifyDnsLength=false skips the 255-octet domain cap ICU4J enforces", + ) + + /** + * Exact WPT inputs where kuri (Unicode 17.0) and ICU4J 77.1 (Unicode 16.0) disagree solely because + * ICU4J's bundled tables still treat a CJK ideograph the input carries (U+32931 or U+32B9A) as + * unassigned. Copied verbatim from [IDNA_CONFORMANCE_CASES] so a corpus regeneration cannot silently + * rename them out from under this set. + */ + private val unicodeVersionSkewInputs: Set = + setOf( + "𲤱20.音.ꡦ1.", // U+32931 label; kuri accepts, ICU4J: DISALLOWED (unassigned) + "xn--20-9802c.xn--0w5a.xn--1-eg4e.", // its ToASCII form, re-decoded and re-checked as an A-label + "xn--9-i0j5967eg3qz.ss", // ToASCII form of the U+32B9A case below, re-checked as an A-label + "𲮚9ꍩ៓.ss", // U+32B9A label; kuri accepts, ICU4J: DISALLOWED (unassigned) + "𲮚9ꍩ៓.SS", // same, uppercase SS label (kuri case-folds before comparing) + ) + + private fun icuToAscii(input: String): Pair> { + val info = IDNA.Info() + val out = StringBuilder() + icu.nameToASCII(input, out, info) + val value = if (info.hasErrors()) null else out.toString() + return value to info.errors + } + + private fun isStructuralOptionDifference(errors: Set): Boolean = + errors.isNotEmpty() && errors.all { it in structuralOptionErrors } + + private fun isExplainedDifference( + input: String, + icuErrors: Set, + ): Boolean = isStructuralOptionDifference(icuErrors) || input in unicodeVersionSkewInputs + + @Test + fun `kuri and ICU4J agree on every WPT case outside the documented differences`() { + val unexplained = + IDNA_CONFORMANCE_CASES.mapNotNull { case -> + val kuriValue = (Idna.domainToAscii(case.input) as? ParseResult.Ok)?.value + val (icuValue, icuErrors) = icuToAscii(case.input) + if (kuriValue == icuValue || isExplainedDifference(case.input, icuErrors)) { + null + } else { + "input=<${case.input}> kuri=<$kuriValue> icu=<$icuValue> icuErrors=$icuErrors" + } + } + assertTrue(unexplained.isEmpty(), "unexplained kuri/ICU4J disagreements:\n${unexplained.joinToString("\n")}") + } + + @Test + fun `every documented Unicode version skew input is a live kuri and ICU4J disagreement`() { + // Ratchet: a kuri or ICU4J Unicode-version bump that resolves the skew makes kuri and ICU4J agree + // again, and this then fails until the now-stale entry is removed from the documented set above. + unicodeVersionSkewInputs.forEach { input -> + val kuriValue = (Idna.domainToAscii(input) as? ParseResult.Ok)?.value + val (icuValue, _) = icuToAscii(input) + assertTrue(kuriValue != icuValue, "expected a live kuri/ICU4J disagreement for <$input>") + } + } + + @Test + fun `the corpus dwarfs the documented Unicode version skew`() { + assertTrue(IDNA_CONFORMANCE_CASES.size > 10 * unicodeVersionSkewInputs.size) + assertTrue(unicodeVersionSkewInputs.isNotEmpty()) + } +} diff --git a/tools/internal/codegen/conformance.go b/tools/internal/codegen/conformance.go index fc9028d..b1a0807 100644 --- a/tools/internal/codegen/conformance.go +++ b/tools/internal/codegen/conformance.go @@ -15,6 +15,10 @@ // of kuri's Idna.domainToAscii is run over the corpus and the inputs it fails // become the tracked set. Because the reference matches kuri exactly, that set // equals kuri's live failing set, which IdnaConformanceTest asserts against. +// +// idnaref is a second implementation of kuri's own algorithm, not an independent +// oracle — see the idnaref package doc. WPT is the independent oracle IdnaConformanceTest +// checks kuri against; idnaref only narrows the residual within whatever WPT doesn't cover. package codegen diff --git a/tools/internal/codegen/main.go b/tools/internal/codegen/main.go index 51dad6f..d40aafa 100644 --- a/tools/internal/codegen/main.go +++ b/tools/internal/codegen/main.go @@ -17,7 +17,8 @@ import ( // usage describes the command line for error messages. const usage = "usage: codegen [--stdout]\n" + - " names: url, idna-mapping, idna-validity, nfc-tables, nfc-test, conformance, setters, percent-encoding\n" + + " names: url, idna-mapping, idna-validity, nfc-tables, nfc-test, conformance, setters, percent-encoding,\n" + + " urlencoded\n" + " conformance also accepts --stdout-data / --stdout-known to print one of its two files" // singleGenerator pairs a single-file generator's source builder with a resolver @@ -27,8 +28,8 @@ type singleGenerator struct { output func() (string, error) } -// singleGenerators is the dispatch table for the five generators that emit one -// fixture each. conformance is handled separately in Main because it emits two +// singleGenerators is the dispatch table for the generators that each emit one +// fixture. conformance is handled separately in Main because it emits two // files and accepts its own stdout selectors. var singleGenerators = map[string]singleGenerator{ "url": { @@ -59,6 +60,10 @@ var singleGenerators = map[string]singleGenerator{ generate: generatePercentEncoding, output: outputPath("kuri", "src", "commonTest", "kotlin", "org", "dexpace", "kuri", "percent", "PercentEncodingTestData.kt"), }, + "urlencoded": { + generate: generateUrlEncoded, + output: outputPath("kuri", "src", "commonTest", "kotlin", "org", "dexpace", "kuri", "query", "UrlEncodedTestData.kt"), + }, } // Main dispatches to the named generator. The generator name is the first diff --git a/tools/internal/codegen/urlencoded.go b/tools/internal/codegen/urlencoded.go new file mode 100644 index 0000000..4e96504 --- /dev/null +++ b/tools/internal/codegen/urlencoded.go @@ -0,0 +1,223 @@ +// Copyright (c) 2026 dexpace and Omar Aljarrah +// SPDX-License-Identifier: MIT + +// The urlencoded generator reads the vendored WPT urlencoded-parser corpus and materializes the +// Kotlin UrlEncodedTestData fixture. There is no standalone JSON resource upstream: the corpus is +// hand-extracted from the array literal at the top of WPT's url/urlencoded-parser.any.js. Only the +// plain input/output pairs feeding that file's "URLSearchParams constructed with" subtest are +// vendored; its two sibling formData() subtests exercise non-UTF-8 charsets (windows-1252, +// shift_jis) that are out of scope, since kuri's form codec is UTF-8-only. Like percentencoding.go, +// no entry carries a lone surrogate, so decoding goes straight through encoding/json rather than +// url.go's/json.go's UTF-16 lossless path. +package codegen + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "unicode/utf16" +) + +// urlEncodedChunkSize is the number of cases per generated builder method, kept well under the +// 64 KB JVM constant-pool limit so no single method overflows. +const urlEncodedChunkSize = 60 + +// urlEncodedPair is one decoded name/value pair of a case's expected output, in appearance order. +type urlEncodedPair struct { + name string + value string +} + +// urlEncodedCase is one in-scope corpus entry: parsing input as application/x-www-form-urlencoded +// must yield output, in order. +type urlEncodedCase struct { + input string + output []urlEncodedPair +} + +// urlEncodedEntry mirrors one array element of urlencoded-parser.json: {input, output: [[name, +// value], ...]}. +type urlEncodedEntry struct { + Input *string `json:"input"` + Output [][]string `json:"output"` +} + +// urlEncodedInputPath returns the absolute path of the vendored WPT corpus. +func urlEncodedInputPath() (string, error) { + root, err := Root() + if err != nil { + return "", err + } + return filepath.Join(root, "tools", "urlencoded", "urlencoded-parser.json"), nil +} + +// generateUrlEncoded reads the corpus and returns the complete Kotlin source string. +func generateUrlEncoded() (string, error) { + path, err := urlEncodedInputPath() + if err != nil { + return "", err + } + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + cases, err := urlEncodedLoadCases(data) + if err != nil { + return "", err + } + return urlEncodedEmitFixture(cases) +} + +// urlEncodedLoadCases decodes the top-level JSON array and keeps only objects carrying an `input` +// key, preserving document order with no sorting or deduplication. The leading documentation +// string (skipped via firstNonSpace, as url.go/percentencoding.go do) is dropped. +func urlEncodedLoadCases(data []byte) ([]urlEncodedCase, error) { + var elements []json.RawMessage + if err := json.Unmarshal(data, &elements); err != nil { + return nil, fmt.Errorf("urlencoded: decoding corpus array: %w", err) + } + cases := make([]urlEncodedCase, 0, len(elements)) + for index, element := range elements { + if firstNonSpace(element) != '{' { + continue // documentation string + } + var entry urlEncodedEntry + if err := json.Unmarshal(element, &entry); err != nil { + return nil, fmt.Errorf("urlencoded: decoding object at index %d: %w", index, err) + } + if entry.Input == nil { + continue + } + output, err := urlEncodedPairs(entry.Output, index) + if err != nil { + return nil, err + } + cases = append(cases, urlEncodedCase{input: *entry.Input, output: output}) + } + return cases, nil +} + +// urlEncodedPairs converts one entry's raw [[name, value], ...] rows into pairs, erroring on any +// row that is not exactly two elements (the shape every corpus row currently has). +func urlEncodedPairs(rows [][]string, index int) ([]urlEncodedPair, error) { + pairs := make([]urlEncodedPair, 0, len(rows)) + for _, row := range rows { + if len(row) != 2 { + return nil, fmt.Errorf("urlencoded: entry %d has an output row with %d elements, want 2", index, len(row)) + } + pairs = append(pairs, urlEncodedPair{name: row[0], value: row[1]}) + } + return pairs, nil +} + +// urlEncodedEmitFixture renders the complete fixture: license header, suppress block, package, +// data class, and the chunked builder object, terminated by exactly one newline. The structure +// mirrors urlEmitFixture/percentEmitFixture. +func urlEncodedEmitFixture(cases []urlEncodedCase) (string, error) { + parts := Chunk(cases, urlEncodedChunkSize) + lines := []string{LicenseHeader, ""} + lines = append(lines, FileSuppressBlock(bulkDataComments, "LongMethod", "LargeClass", "MatchingDeclarationName")...) + lines = append(lines, + "package org.dexpace.kuri.query", + "", + ) + lines = append(lines, urlEncodedDataClassLines()...) + lines = append(lines, + "", + "// Generated by ./gradlew generateUrlEncodedTestData from the WPT urlencoded-parser corpus.", + "// Chunked into small builders so no single method exceeds the 64 KB JVM limit.", + "private object UrlEncodedCaseData {", + ) + lines = append(lines, urlEncodedAllAccessorLines(parts)...) + for index, part := range parts { + lines = append(lines, "") + lines = append(lines, fmt.Sprintf(" private fun part%d(): List =", index)) + lines = append(lines, " listOf(") + for _, current := range part { + caseLines, err := urlEncodedCaseLines(current) + if err != nil { + return "", err + } + lines = append(lines, caseLines...) + } + lines = append(lines, " )") + } + lines = append(lines, + "}", + "", + "/** Every WPT `urlencoded-parser.json` case (objects carrying an `input`). */", + "internal val URL_ENCODED_TEST_CASES: List = UrlEncodedCaseData.all()", + ) + return JoinLines(lines), nil +} + +// urlEncodedAllAccessorLines renders the `fun all(): List = ...` accessor, +// mirroring percentencoding.go's allAccessorLines: with a single chunk (the whole corpus is a +// handful of cases) PartSumExpression collapses to one term and, when the combined signature and +// body still fit MaxCols, ktlint's function-signature rule requires that single line rather than a +// wrap; with more than one chunk the body spans multiple lines and must wrap regardless. +func urlEncodedAllAccessorLines(parts [][]urlEncodedCase) []string { + signature := " fun all(): List =" + body := PartSumExpression(len(parts)) + if !strings.Contains(body, "\n") { + combined := signature + " " + body + if len(combined) <= MaxCols { + return []string{combined} + } + } + return []string{signature, " " + body} +} + +// urlEncodedCaseLines renders one UrlEncodedCase constructor call as indented named arguments. +func urlEncodedCaseLines(c urlEncodedCase) ([]string, error) { + lines := []string{" UrlEncodedCase("} + lines = append(lines, FieldLines("input", utf16.Encode([]rune(c.input)), fieldIndent)...) + outputLines, err := urlEncodedOutputLines(c.output) + if err != nil { + return nil, err + } + lines = append(lines, outputLines...) + lines = append(lines, " ),") + return lines, nil +} + +// urlEncodedOutputLines renders the `output = listOf("name" to "value", ...),` field (or +// `output = emptyList(),` when the case yields no pairs). ktlint requires a multiline call on the +// right of an `=` to start on its own line (as expectedFieldLines notes for `mapOf`), so `listOf(` +// cannot trail `output =` on the same line. Every corpus name/value is short enough that one pair +// fits one line comfortably within MaxCols; a pair that would not is a genuine generator gap, so it +// errors rather than silently emitting an over-long line for ktlint to catch. +func urlEncodedOutputLines(pairs []urlEncodedPair) ([]string, error) { + pad := strings.Repeat(" ", fieldIndent) + if len(pairs) == 0 { + return []string{pad + "output = emptyList(),"}, nil + } + listPad := strings.Repeat(" ", fieldIndent+4) + entryPad := strings.Repeat(" ", fieldIndent+8) + lines := []string{pad + "output =", listPad + "listOf("} + for _, pair := range pairs { + line := entryPad + KotlinString(pair.name) + " to " + KotlinString(pair.value) + "," + if len(line) > MaxCols { + return nil, fmt.Errorf("urlencoded: output pair %q/%q exceeds %d columns", pair.name, pair.value, MaxCols) + } + lines = append(lines, line) + } + lines = append(lines, listPad+"),") + return lines, nil +} + +// urlEncodedDataClassLines is the fixed KDoc + UrlEncodedCase data-class declaration. +func urlEncodedDataClassLines() []string { + return []string{ + "/**", + " * One WPT `urlencoded-parser.json` case: parsing [input] as `application/x-www-form-urlencoded`", + " * must yield [output], the ordered list of decoded `(name, value)` pairs.", + " */", + "internal data class UrlEncodedCase(", + " val input: String,", + " val output: List>,", + ")", + } +} diff --git a/tools/internal/idnaref/reference.go b/tools/internal/idnaref/reference.go index 517922d..76d9349 100644 --- a/tools/internal/idnaref/reference.go +++ b/tools/internal/idnaref/reference.go @@ -6,6 +6,15 @@ // tables decode. The codegen conformance generator runs it over the WPT corpora // to derive the tracked known-failures baseline. It depends only on internal/ucd // for its table data. +// +// idnaref is not an independent oracle: it re-derives kuri's own algorithm as a +// second implementation, so a conceptual bug shared between the Kotlin engine and +// this port (e.g. a shared misreading of the UTS-46 algorithm) would reproduce +// identically in both and pass the ratchet undetected. Its value is catching +// porting slips between the two — a divergence that is NOT shared — not verifying +// the algorithm itself. WPT is the real independent oracle here: IdnaConformanceTest +// checks every corpus case directly against it, and idnaref only narrows the +// residual within whatever WPT doesn't already exercise. package idnaref import ( diff --git a/tools/urlencoded/urlencoded-parser.json b/tools/urlencoded/urlencoded-parser.json new file mode 100644 index 0000000..39b4209 --- /dev/null +++ b/tools/urlencoded/urlencoded-parser.json @@ -0,0 +1,352 @@ +[ + "Vendored from the WPT array literal at the top of url/urlencoded-parser.any.js (web-platform-tests/wpt, master branch, fetched 2026-07-13). Only the plain input/output pairs used by the \"URLSearchParams constructed with\" subtest are kept; the two per-entry formData() subtests exercise non-UTF-8 charsets (windows-1252, shift_jis) that are out of scope for kuri, which is UTF-8-only.", + { + "input": "test", + "output": [ + [ + "test", + "" + ] + ] + }, + { + "input": "test=", + "output": [ + [ + "test", + "" + ] + ] + }, + { + "input": "%EF%BB%BFtest=%EF%BB%BF", + "output": [ + [ + "test", + "" + ] + ] + }, + { + "input": "%EF%BF%BF=%EF%BF%BF", + "output": [ + [ + "￿", + "￿" + ] + ] + }, + { + "input": "%FE%FF", + "output": [ + [ + "��", + "" + ] + ] + }, + { + "input": "%FF%FE", + "output": [ + [ + "��", + "" + ] + ] + }, + { + "input": "†&†=x", + "output": [ + [ + "†", + "" + ], + [ + "†", + "x" + ] + ] + }, + { + "input": "%C2", + "output": [ + [ + "�", + "" + ] + ] + }, + { + "input": "%C2x", + "output": [ + [ + "�x", + "" + ] + ] + }, + { + "input": "_charset_=windows-1252&test=%C2x", + "output": [ + [ + "_charset_", + "windows-1252" + ], + [ + "test", + "�x" + ] + ] + }, + { + "input": "", + "output": [] + }, + { + "input": "a", + "output": [ + [ + "a", + "" + ] + ] + }, + { + "input": "a=b", + "output": [ + [ + "a", + "b" + ] + ] + }, + { + "input": "a=", + "output": [ + [ + "a", + "" + ] + ] + }, + { + "input": "=b", + "output": [ + [ + "", + "b" + ] + ] + }, + { + "input": "&", + "output": [] + }, + { + "input": "&a", + "output": [ + [ + "a", + "" + ] + ] + }, + { + "input": "a&", + "output": [ + [ + "a", + "" + ] + ] + }, + { + "input": "a&a", + "output": [ + [ + "a", + "" + ], + [ + "a", + "" + ] + ] + }, + { + "input": "a&b&c", + "output": [ + [ + "a", + "" + ], + [ + "b", + "" + ], + [ + "c", + "" + ] + ] + }, + { + "input": "a=b&c=d", + "output": [ + [ + "a", + "b" + ], + [ + "c", + "d" + ] + ] + }, + { + "input": "a=b&c=d&", + "output": [ + [ + "a", + "b" + ], + [ + "c", + "d" + ] + ] + }, + { + "input": "&&&a=b&&&&c=d&", + "output": [ + [ + "a", + "b" + ], + [ + "c", + "d" + ] + ] + }, + { + "input": "a=a&a=b&a=c", + "output": [ + [ + "a", + "a" + ], + [ + "a", + "b" + ], + [ + "a", + "c" + ] + ] + }, + { + "input": "a==a", + "output": [ + [ + "a", + "=a" + ] + ] + }, + { + "input": "a=a+b+c+d", + "output": [ + [ + "a", + "a b c d" + ] + ] + }, + { + "input": "%=a", + "output": [ + [ + "%", + "a" + ] + ] + }, + { + "input": "%a=a", + "output": [ + [ + "%a", + "a" + ] + ] + }, + { + "input": "%a_=a", + "output": [ + [ + "%a_", + "a" + ] + ] + }, + { + "input": "%61=a", + "output": [ + [ + "a", + "a" + ] + ] + }, + { + "input": "%61+%4d%4D=", + "output": [ + [ + "a MM", + "" + ] + ] + }, + { + "input": "id=0&value=%", + "output": [ + [ + "id", + "0" + ], + [ + "value", + "%" + ] + ] + }, + { + "input": "b=%2sf%2a", + "output": [ + [ + "b", + "%2sf*" + ] + ] + }, + { + "input": "b=%2%2af%2a", + "output": [ + [ + "b", + "%2*f*" + ] + ] + }, + { + "input": "b=%%2a", + "output": [ + [ + "b", + "%*" + ] + ] + } +]