zerotrie

unicode-org · Sep 24, 2024 · 0ddcc6d · 0ddcc6d
1 parent 38e6949
commit 0ddcc6d
Show file tree

Hide file tree

Showing 13 changed files with 221 additions and 451 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/components/experimental/src/transliterate/compile/mod.rs b/components/experimental/src/transliterate/compile/mod.rs
@@ -184,7 +184,7 @@ impl RuleCollection {
             + DataProvider<ExtenderV1Marker>
             + DataProvider<GraphemeBaseV1Marker>
             + DataProvider<GraphemeClusterBreakV1Marker>
-            + DataProvider<GraphemeClusterBreakNameToValueV1Marker>
+            + DataProvider<GraphemeClusterBreakNameToValueV2Marker>
             + DataProvider<GraphemeExtendV1Marker>
             + DataProvider<HexDigitV1Marker>
             + DataProvider<IdsBinaryOperatorV1Marker>
@@ -203,7 +203,7 @@ impl RuleCollection {
             + DataProvider<RadicalV1Marker>
             + DataProvider<RegionalIndicatorV1Marker>
             + DataProvider<SentenceBreakV1Marker>
-            + DataProvider<SentenceBreakNameToValueV1Marker>
+            + DataProvider<SentenceBreakNameToValueV2Marker>
             + DataProvider<SentenceTerminalV1Marker>
             + DataProvider<SoftDottedV1Marker>
             + DataProvider<TerminalPunctuationV1Marker>
@@ -212,11 +212,11 @@ impl RuleCollection {
             + DataProvider<VariationSelectorV1Marker>
             + DataProvider<WhiteSpaceV1Marker>
             + DataProvider<WordBreakV1Marker>
-            + DataProvider<WordBreakNameToValueV1Marker>
+            + DataProvider<WordBreakNameToValueV2Marker>
             + DataProvider<XidContinueV1Marker>
-            + DataProvider<GeneralCategoryMaskNameToValueV1Marker>
+            + DataProvider<GeneralCategoryMaskNameToValueV2Marker>
             + DataProvider<GeneralCategoryV1Marker>
-            + DataProvider<ScriptNameToValueV1Marker>
+            + DataProvider<ScriptNameToValueV2Marker>
             + DataProvider<ScriptV1Marker>
             + DataProvider<ScriptWithExtensionsPropertyV1Marker>
             + DataProvider<XidStartV1Marker>,
@@ -277,7 +277,7 @@ where
         + DataProvider<ExtenderV1Marker>
         + DataProvider<GraphemeBaseV1Marker>
         + DataProvider<GraphemeClusterBreakV1Marker>
-        + DataProvider<GraphemeClusterBreakNameToValueV1Marker>
+        + DataProvider<GraphemeClusterBreakNameToValueV2Marker>
         + DataProvider<GraphemeExtendV1Marker>
         + DataProvider<HexDigitV1Marker>
         + DataProvider<IdsBinaryOperatorV1Marker>
@@ -296,7 +296,7 @@ where
         + DataProvider<RadicalV1Marker>
         + DataProvider<RegionalIndicatorV1Marker>
         + DataProvider<SentenceBreakV1Marker>
-        + DataProvider<SentenceBreakNameToValueV1Marker>
+        + DataProvider<SentenceBreakNameToValueV2Marker>
         + DataProvider<SentenceTerminalV1Marker>
         + DataProvider<SoftDottedV1Marker>
         + DataProvider<TerminalPunctuationV1Marker>
@@ -305,11 +305,11 @@ where
         + DataProvider<VariationSelectorV1Marker>
         + DataProvider<WhiteSpaceV1Marker>
         + DataProvider<WordBreakV1Marker>
-        + DataProvider<WordBreakNameToValueV1Marker>
+        + DataProvider<WordBreakNameToValueV2Marker>
         + DataProvider<XidContinueV1Marker>
-        + DataProvider<GeneralCategoryMaskNameToValueV1Marker>
+        + DataProvider<GeneralCategoryMaskNameToValueV2Marker>
         + DataProvider<GeneralCategoryV1Marker>
-        + DataProvider<ScriptNameToValueV1Marker>
+        + DataProvider<ScriptNameToValueV2Marker>
         + DataProvider<ScriptV1Marker>
         + DataProvider<ScriptWithExtensionsPropertyV1Marker>
         + DataProvider<XidStartV1Marker>,
@@ -438,7 +438,7 @@ where
         + DataProvider<ExtenderV1Marker>
         + DataProvider<GraphemeBaseV1Marker>
         + DataProvider<GraphemeClusterBreakV1Marker>
-        + DataProvider<GraphemeClusterBreakNameToValueV1Marker>
+        + DataProvider<GraphemeClusterBreakNameToValueV2Marker>
         + DataProvider<GraphemeExtendV1Marker>
         + DataProvider<HexDigitV1Marker>
         + DataProvider<IdsBinaryOperatorV1Marker>
@@ -457,7 +457,7 @@ where
         + DataProvider<RadicalV1Marker>
         + DataProvider<RegionalIndicatorV1Marker>
         + DataProvider<SentenceBreakV1Marker>
-        + DataProvider<SentenceBreakNameToValueV1Marker>
+        + DataProvider<SentenceBreakNameToValueV2Marker>
         + DataProvider<SentenceTerminalV1Marker>
         + DataProvider<SoftDottedV1Marker>
         + DataProvider<TerminalPunctuationV1Marker>
@@ -466,11 +466,11 @@ where
         + DataProvider<VariationSelectorV1Marker>
         + DataProvider<WhiteSpaceV1Marker>
         + DataProvider<WordBreakV1Marker>
-        + DataProvider<WordBreakNameToValueV1Marker>
+        + DataProvider<WordBreakNameToValueV2Marker>
         + DataProvider<XidContinueV1Marker>
-        + DataProvider<GeneralCategoryMaskNameToValueV1Marker>
+        + DataProvider<GeneralCategoryMaskNameToValueV2Marker>
         + DataProvider<GeneralCategoryV1Marker>
-        + DataProvider<ScriptNameToValueV1Marker>
+        + DataProvider<ScriptNameToValueV2Marker>
         + DataProvider<ScriptV1Marker>
         + DataProvider<ScriptWithExtensionsPropertyV1Marker>
         + DataProvider<XidStartV1Marker>,

diff --git a/components/experimental/src/transliterate/compile/parse.rs b/components/experimental/src/transliterate/compile/parse.rs
@@ -287,7 +287,7 @@ where
         + DataProvider<ExtenderV1Marker>
         + DataProvider<GraphemeBaseV1Marker>
         + DataProvider<GraphemeClusterBreakV1Marker>
-        + DataProvider<GraphemeClusterBreakNameToValueV1Marker>
+        + DataProvider<GraphemeClusterBreakNameToValueV2Marker>
         + DataProvider<GraphemeExtendV1Marker>
         + DataProvider<HexDigitV1Marker>
         + DataProvider<IdsBinaryOperatorV1Marker>
@@ -306,7 +306,7 @@ where
         + DataProvider<RadicalV1Marker>
         + DataProvider<RegionalIndicatorV1Marker>
         + DataProvider<SentenceBreakV1Marker>
-        + DataProvider<SentenceBreakNameToValueV1Marker>
+        + DataProvider<SentenceBreakNameToValueV2Marker>
         + DataProvider<SentenceTerminalV1Marker>
         + DataProvider<SoftDottedV1Marker>
         + DataProvider<TerminalPunctuationV1Marker>
@@ -315,11 +315,11 @@ where
         + DataProvider<VariationSelectorV1Marker>
         + DataProvider<WhiteSpaceV1Marker>
         + DataProvider<WordBreakV1Marker>
-        + DataProvider<WordBreakNameToValueV1Marker>
+        + DataProvider<WordBreakNameToValueV2Marker>
         + DataProvider<XidContinueV1Marker>
-        + DataProvider<GeneralCategoryMaskNameToValueV1Marker>
+        + DataProvider<GeneralCategoryMaskNameToValueV2Marker>
         + DataProvider<GeneralCategoryV1Marker>
-        + DataProvider<ScriptNameToValueV1Marker>
+        + DataProvider<ScriptNameToValueV2Marker>
         + DataProvider<ScriptV1Marker>
         + DataProvider<ScriptWithExtensionsPropertyV1Marker>
         + DataProvider<XidStartV1Marker>,

diff --git a/components/experimental/src/unicodeset_parse/parse.rs b/components/experimental/src/unicodeset_parse/parse.rs
@@ -451,7 +451,7 @@ where
         + DataProvider<ExtenderV1Marker>
         + DataProvider<GraphemeBaseV1Marker>
         + DataProvider<GraphemeClusterBreakV1Marker>
-        + DataProvider<GraphemeClusterBreakNameToValueV1Marker>
+        + DataProvider<GraphemeClusterBreakNameToValueV2Marker>
         + DataProvider<GraphemeExtendV1Marker>
         + DataProvider<HexDigitV1Marker>
         + DataProvider<IdsBinaryOperatorV1Marker>
@@ -470,7 +470,7 @@ where
         + DataProvider<RadicalV1Marker>
         + DataProvider<RegionalIndicatorV1Marker>
         + DataProvider<SentenceBreakV1Marker>
-        + DataProvider<SentenceBreakNameToValueV1Marker>
+        + DataProvider<SentenceBreakNameToValueV2Marker>
         + DataProvider<SentenceTerminalV1Marker>
         + DataProvider<SoftDottedV1Marker>
         + DataProvider<TerminalPunctuationV1Marker>
@@ -479,11 +479,11 @@ where
         + DataProvider<VariationSelectorV1Marker>
         + DataProvider<WhiteSpaceV1Marker>
         + DataProvider<WordBreakV1Marker>
-        + DataProvider<WordBreakNameToValueV1Marker>
+        + DataProvider<WordBreakNameToValueV2Marker>
         + DataProvider<XidContinueV1Marker>
-        + DataProvider<GeneralCategoryMaskNameToValueV1Marker>
+        + DataProvider<GeneralCategoryMaskNameToValueV2Marker>
         + DataProvider<GeneralCategoryV1Marker>
-        + DataProvider<ScriptNameToValueV1Marker>
+        + DataProvider<ScriptNameToValueV2Marker>
         + DataProvider<ScriptV1Marker>
         + DataProvider<ScriptWithExtensionsPropertyV1Marker>
         + DataProvider<XidStartV1Marker>,
@@ -1330,7 +1330,7 @@ where
                 .map_err(|_| PEK::Internal)?;
         let gc_value = name_map
             .as_borrowed()
-            .get_loose(name)
+            .get(name)
             .ok_or(PEK::UnknownProperty)?;
         // TODO(#3550): This could be cached; does not depend on name.
         let set = CodePointMapData::<GeneralCategory>::try_new_unstable(self.property_provider)
@@ -1347,7 +1347,7 @@ where
             .map_err(|_| PEK::Internal)?;
         name_map
             .as_borrowed()
-            .get_loose(name)
+            .get(name)
             .ok_or(PEK::UnknownProperty.into())
     }
 
@@ -1384,10 +1384,7 @@ where
         let parser =
             PropertyParser::<GraphemeClusterBreak>::try_new_unstable(self.property_provider)
                 .map_err(|_| PEK::Internal)?;
-        let gcb_value = parser
-            .as_borrowed()
-            .get_loose(name)
-            .ok_or(PEK::UnknownProperty)?;
+        let gcb_value = parser.as_borrowed().get(name).ok_or(PEK::UnknownProperty)?;
         // TODO(#3550): This could be cached; does not depend on name.
         let property_map =
             CodePointMapData::<GraphemeClusterBreak>::try_new_unstable(self.property_provider)
@@ -1400,10 +1397,7 @@ where
     fn try_load_sentence_break_set(&mut self, name: &str) -> Result<()> {
         let parser = PropertyParser::<SentenceBreak>::try_new_unstable(self.property_provider)
             .map_err(|_| PEK::Internal)?;
-        let sb_value = parser
-            .as_borrowed()
-            .get_loose(name)
-            .ok_or(PEK::UnknownProperty)?;
+        let sb_value = parser.as_borrowed().get(name).ok_or(PEK::UnknownProperty)?;
         // TODO(#3550): This could be cached; does not depend on name.
         let property_map =
             CodePointMapData::<SentenceBreak>::try_new_unstable(self.property_provider)
@@ -1416,10 +1410,7 @@ where
     fn try_load_word_break_set(&mut self, name: &str) -> Result<()> {
         let parser = PropertyParser::<WordBreak>::try_new_unstable(self.property_provider)
             .map_err(|_| PEK::Internal)?;
-        let wb_value = parser
-            .as_borrowed()
-            .get_loose(name)
-            .ok_or(PEK::UnknownProperty)?;
+        let wb_value = parser.as_borrowed().get(name).ok_or(PEK::UnknownProperty)?;
         // TODO(#3550): This could be cached; does not depend on name.
         let property_map = CodePointMapData::<WordBreak>::try_new_unstable(self.property_provider)
             .map_err(|_| PEK::Internal)?;
@@ -1582,7 +1573,7 @@ where
         + DataProvider<ExtenderV1Marker>
         + DataProvider<GraphemeBaseV1Marker>
         + DataProvider<GraphemeClusterBreakV1Marker>
-        + DataProvider<GraphemeClusterBreakNameToValueV1Marker>
+        + DataProvider<GraphemeClusterBreakNameToValueV2Marker>
         + DataProvider<GraphemeExtendV1Marker>
         + DataProvider<HexDigitV1Marker>
         + DataProvider<IdsBinaryOperatorV1Marker>
@@ -1601,7 +1592,7 @@ where
         + DataProvider<RadicalV1Marker>
         + DataProvider<RegionalIndicatorV1Marker>
         + DataProvider<SentenceBreakV1Marker>
-        + DataProvider<SentenceBreakNameToValueV1Marker>
+        + DataProvider<SentenceBreakNameToValueV2Marker>
         + DataProvider<SentenceTerminalV1Marker>
         + DataProvider<SoftDottedV1Marker>
         + DataProvider<TerminalPunctuationV1Marker>
@@ -1610,11 +1601,11 @@ where
         + DataProvider<VariationSelectorV1Marker>
         + DataProvider<WhiteSpaceV1Marker>
         + DataProvider<WordBreakV1Marker>
-        + DataProvider<WordBreakNameToValueV1Marker>
+        + DataProvider<WordBreakNameToValueV2Marker>
         + DataProvider<XidContinueV1Marker>
-        + DataProvider<GeneralCategoryMaskNameToValueV1Marker>
+        + DataProvider<GeneralCategoryMaskNameToValueV2Marker>
         + DataProvider<GeneralCategoryV1Marker>
-        + DataProvider<ScriptNameToValueV1Marker>
+        + DataProvider<ScriptNameToValueV2Marker>
         + DataProvider<ScriptV1Marker>
         + DataProvider<ScriptWithExtensionsPropertyV1Marker>
         + DataProvider<XidStartV1Marker>,
@@ -1696,7 +1687,7 @@ where
         + DataProvider<ExtenderV1Marker>
         + DataProvider<GraphemeBaseV1Marker>
         + DataProvider<GraphemeClusterBreakV1Marker>
-        + DataProvider<GraphemeClusterBreakNameToValueV1Marker>
+        + DataProvider<GraphemeClusterBreakNameToValueV2Marker>
         + DataProvider<GraphemeExtendV1Marker>
         + DataProvider<HexDigitV1Marker>
         + DataProvider<IdsBinaryOperatorV1Marker>
@@ -1715,7 +1706,7 @@ where
         + DataProvider<RadicalV1Marker>
         + DataProvider<RegionalIndicatorV1Marker>
         + DataProvider<SentenceBreakV1Marker>
-        + DataProvider<SentenceBreakNameToValueV1Marker>
+        + DataProvider<SentenceBreakNameToValueV2Marker>
         + DataProvider<SentenceTerminalV1Marker>
         + DataProvider<SoftDottedV1Marker>
         + DataProvider<TerminalPunctuationV1Marker>
@@ -1724,11 +1715,11 @@ where
         + DataProvider<VariationSelectorV1Marker>
         + DataProvider<WhiteSpaceV1Marker>
         + DataProvider<WordBreakV1Marker>
-        + DataProvider<WordBreakNameToValueV1Marker>
+        + DataProvider<WordBreakNameToValueV2Marker>
         + DataProvider<XidContinueV1Marker>
-        + DataProvider<GeneralCategoryMaskNameToValueV1Marker>
+        + DataProvider<GeneralCategoryMaskNameToValueV2Marker>
         + DataProvider<GeneralCategoryV1Marker>
-        + DataProvider<ScriptNameToValueV1Marker>
+        + DataProvider<ScriptNameToValueV2Marker>
         + DataProvider<ScriptV1Marker>
         + DataProvider<ScriptWithExtensionsPropertyV1Marker>
         + DataProvider<XidStartV1Marker>,

diff --git a/components/properties/Cargo.toml b/components/properties/Cargo.toml
@@ -23,7 +23,8 @@ all-features = true
 displaydoc = { workspace = true }
 icu_collections = { workspace = true }
 icu_provider = { workspace = true, features = ["macros"] }
-zerovec = { workspace = true, features = ["derive"] }
+zerovec = { workspace = true, features = ["derive", "yoke"] }
+zerotrie = { workspace = true, features = ["yoke", "zerofrom"] }
 databake = { workspace = true, features = ["derive"], optional = true}
 serde = { workspace = true, features = ["derive", "alloc"], optional = true }
 icu_locale_core = { workspace = true, features = ["zerovec"] }
@@ -39,7 +40,7 @@ icu = { path = "../../components/icu", default-features = false }
 [features]
 default = ["compiled_data"]
 std = ["icu_collections/std", "icu_provider/std"]
-serde = ["dep:serde", "icu_locale_core/serde", "potential_utf/serde", "zerovec/serde", "icu_collections/serde", "icu_provider/serde"]
-datagen = ["serde", "dep:databake", "potential_utf/databake", "zerovec/databake", "icu_collections/databake", "icu_locale_core/databake"]
+serde = ["dep:serde", "icu_locale_core/serde", "potential_utf/serde", "zerovec/serde", "icu_collections/serde", "icu_provider/serde", "zerotrie/serde"]
+datagen = ["serde", "dep:databake", "potential_utf/databake", "zerovec/databake", "icu_collections/databake", "icu_locale_core/databake", "zerotrie/databake"]
 unicode_bidi = [ "dep:unicode-bidi" ]
 compiled_data = ["dep:icu_properties_data"]