Skip to content

Commit

Permalink
Use script fallback in collator (#5743)
Browse files Browse the repository at this point in the history 
Fixes #3867?
  • Loading branch information
@robertbastian
robertbastian authored Oct 29, 2024
1 parent 4865965 commit 88a1a54
Show file tree
Hide file tree
Showing 26 changed files with 407,614 additions and 70 deletions.
15 changes: 5 additions & 10 deletions components/collator/src/provider.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,16 +117,11 @@ fn data_ce_to_primary(data_ce: u64, c: char) -> u32 {
/// to be stable, their Rust representation might not be. Use with caution.
/// </div>
#[icu_provider::data_struct(
marker(
CollationRootV1Marker,
"collator/root@1",
singleton,
),
marker(CollationRootV1Marker, "collator/root@1", singleton),
marker(
CollationTailoringV1Marker,
"collator/tailoring@1",
// TODO(#3867): Use script fallback
fallback_by = "language",
fallback_by = "script",
attributes_domain = "collator",
)
)]
Expand Down Expand Up @@ -243,7 +238,7 @@ impl<'data> CollationDataV1<'data> {
#[icu_provider::data_struct(marker(
CollationDiacriticsV1Marker,
"collator/dia@1",
fallback_by = "language",
fallback_by = "script",
attributes_domain = "collator",
))]
#[derive(Debug, PartialEq, Clone)]
Expand Down Expand Up @@ -288,7 +283,7 @@ pub struct CollationJamoV1<'data> {
#[icu_provider::data_struct(marker(
CollationReorderingV1Marker,
"collator/reord@1",
fallback_by = "language",
fallback_by = "script",
attributes_domain = "collator",
))]
#[derive(Debug, PartialEq, Clone)]
Expand Down Expand Up @@ -378,7 +373,7 @@ impl CollationReorderingV1<'_> {
#[icu_provider::data_struct(marker(
CollationMetadataV1Marker,
"collator/meta@1",
fallback_by = "language",
fallback_by = "script",
attributes_domain = "collator",
))]
#[derive(Debug, PartialEq, Clone, Copy)]
Expand Down
10 changes: 5 additions & 5 deletions components/locale/src/fallback/algorithms.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ impl LocaleFallbackIteratorInner<'_> {
return;
}
// 8. Remove language+script
debug_assert!(!locale.language.is_default()); // don't call .step() on und
debug_assert!(!locale.language.is_default() || locale.script.is_some()); // don't call .step() on und
locale.script = None;
locale.language = Language::UND;
}
Expand Down Expand Up @@ -415,28 +415,28 @@ mod tests {
// The fallback algorithm does not visit the language-script bundle when the
// script is the default for the language
expected_language_chain: &["zh-CN", "zh"],
expected_script_chain: &["zh-CN", "zh", "und-Hans"],
expected_script_chain: &["zh-CN", "zh", "und-Hans", "und-Hani"],
expected_region_chain: &["zh-CN", "und-CN"],
},
TestCase {
input: "zh-TW",
requires_data: true,
expected_language_chain: &["zh-TW", "zh-Hant"],
expected_script_chain: &["zh-TW", "zh-Hant", "und-Hant"],
expected_script_chain: &["zh-TW", "zh-Hant", "und-Hant", "und-Hani"],
expected_region_chain: &["zh-TW", "und-TW"],
},
TestCase {
input: "yue-HK",
requires_data: true,
expected_language_chain: &["yue-HK", "yue"],
expected_script_chain: &["yue-HK", "yue", "und-Hant"],
expected_script_chain: &["yue-HK", "yue", "und-Hant", "und-Hani"],
expected_region_chain: &["yue-HK", "und-HK"],
},
TestCase {
input: "yue-HK",
requires_data: true,
expected_language_chain: &["yue-HK", "yue"],
expected_script_chain: &["yue-HK", "yue", "und-Hant"],
expected_script_chain: &["yue-HK", "yue", "und-Hant", "und-Hani"],
expected_region_chain: &["yue-HK", "und-HK"],
},
TestCase {
Expand Down
4 changes: 2 additions & 2 deletions provider/data/collator/data/collation_metadata_v1_marker.rs.data
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions provider/data/collator/data/collation_reordering_v1_marker.rs.data
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions provider/data/collator/data/collation_tailoring_v1_marker.rs.data
View file

Large diffs are not rendered by default.

36 changes: 21 additions & 15 deletions provider/data/collator/fingerprints.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ collator/dia@1, und, 182B, 160B, 8ace760351a33687
collator/dia@1, vi, 182B, 160B, 6b057eec42bee512
collator/dia@1, vi/trad, -> vi
collator/jamo@1, <singleton>, 1048B, 1026B, ccbfc88a6d3a411e
collator/meta@1, <lookup>, 520B, 108 identifiers
collator/meta@1, <lookup>, 537B, 110 identifiers
collator/meta@1, <total>, 36B, 12B, 9 unique payloads
collator/meta@1, af, 4B, 1B, 9aab82d56f3b362e
collator/meta@1, am, 4B, 1B, d1dcef7af54c8272
Expand Down Expand Up @@ -103,6 +103,12 @@ collator/meta@1, tr, -> af
collator/meta@1, ug, -> ar
collator/meta@1, uk, -> ar
collator/meta@1, und, 4B, 1B, 667dd5401e6fd800
collator/meta@1, und-Hani/pinyin, -> ar
collator/meta@1, und-Hani/stroke, -> ar
collator/meta@1, und-Hani/unihan, -> ar
collator/meta@1, und-Hani/zhuyin, -> ar
collator/meta@1, und-Hans, -> ar
collator/meta@1, und-Hant, -> ar
collator/meta@1, und/emoji, -> und
collator/meta@1, und/eor, -> und
collator/meta@1, ur, -> ar
Expand All @@ -111,12 +117,8 @@ collator/meta@1, vi, -> ee
collator/meta@1, vi/trad, -> ee
collator/meta@1, wo, -> af
collator/meta@1, yo, -> af
collator/meta@1, zh, -> ar
collator/meta@1, zh/stroke, -> ar
collator/meta@1, zh/unihan, -> ar
collator/meta@1, zh/zhuyin, -> ar
collator/prim@1, <singleton>, 40B, 10B, c72898bdaad773b1
collator/reord@1, <lookup>, 293B, 56 identifiers
collator/reord@1, <lookup>, 314B, 58 identifiers
collator/reord@1, <total>, 9464B, 8024B, 30 unique payloads
collator/reord@1, am, 320B, 272B, 3989bb25c1c894c
collator/reord@1, ar, 312B, 264B, 44ac22224ef1784d
Expand Down Expand Up @@ -169,13 +171,15 @@ collator/reord@1, te, 316B, 268B, 53be041ee9b02cf9
collator/reord@1, th, 312B, 264B, 69513292284ea3de
collator/reord@1, ug, -> ar
collator/reord@1, uk, -> be
collator/reord@1, und-Hani/pinyin, 312B, 264B, 699fda2876be5c76
collator/reord@1, und-Hani/stroke, 324B, 276B, f800a05acd7a58c1
collator/reord@1, und-Hani/unihan, -> und-Hani/stroke
collator/reord@1, und-Hani/zhuyin, -> und-Hani/stroke
collator/reord@1, und-Hans, -> und-Hani/pinyin
collator/reord@1, und-Hant, -> und-Hani/stroke
collator/reord@1, ur, -> ar
collator/reord@1, zh, 312B, 264B, 699fda2876be5c76
collator/reord@1, zh/stroke, 324B, 276B, f800a05acd7a58c1
collator/reord@1, zh/unihan, -> zh/stroke
collator/reord@1, zh/zhuyin, -> zh/stroke
collator/root@1, <singleton>, 129820B, 129703B, 7e63bc9ada35e94
collator/tailoring@1, <lookup>, 577B, 98 identifiers
collator/tailoring@1, <lookup>, 596B, 100 identifiers
collator/tailoring@1, <total>, 950740B, 939172B, 91 unique payloads
collator/tailoring@1, af, 1416B, 1288B, bdc4ac64cc63ff0f
collator/tailoring@1, ar, 8392B, 8267B, fb18ec596d8ddece
Expand Down Expand Up @@ -263,6 +267,12 @@ collator/tailoring@1, to, 1716B, 1589B, d22e1e0b0111552a
collator/tailoring@1, tr, 1548B, 1421B, f4f47fb5a2c0003
collator/tailoring@1, ug, 1900B, 1772B, 677d97f2d30caaf0
collator/tailoring@1, uk, 1960B, 1832B, d0864a679db986ec
collator/tailoring@1, und-Hani/pinyin, 104866B, 104742B, 268c68ad6387f5ee
collator/tailoring@1, und-Hani/stroke, 174942B, 174818B, 9d2ad0f336b435d6
collator/tailoring@1, und-Hani/unihan, 4116B, 3990B, c41f1c0b687c1259
collator/tailoring@1, und-Hani/zhuyin, 129206B, 129082B, 40a2f9a76f043f9d
collator/tailoring@1, und-Hans, -> und-Hani/pinyin
collator/tailoring@1, und-Hant, -> und-Hani/stroke
collator/tailoring@1, und/emoji, 30910B, 30784B, 9c1b9376d11f6436
collator/tailoring@1, und/eor, 6268B, 6140B, 96ca4327c318259f
collator/tailoring@1, ur, 2572B, 2445B, 2ad77b180d7824d9
Expand All @@ -271,7 +281,3 @@ collator/tailoring@1, vi, 1536B, 1409B, 7643d941a22cc0b2
collator/tailoring@1, vi/trad, 1880B, 1753B, 45a30adb3ba0eb0f
collator/tailoring@1, wo, 1528B, 1400B, 344d707597bb2c32
collator/tailoring@1, yo, 1274B, 1146B, f8f8dcc72fee6ec1
collator/tailoring@1, zh, 104866B, 104742B, 268c68ad6387f5ee
collator/tailoring@1, zh/stroke, 174942B, 174818B, 9d2ad0f336b435d6
collator/tailoring@1, zh/unihan, 4116B, 3990B, c41f1c0b687c1259
collator/tailoring@1, zh/zhuyin, 129206B, 129082B, 40a2f9a76f043f9d
Loading

0 comments on commit 88a1a54

Please sign in to comment.