Skip to content

Commit

Permalink
checksum
Browse files Browse the repository at this point in the history
  • Loading branch information
@robertbastian
robertbastian committed Jan 29, 2025
1 parent db77c5b commit 009cf1a
Show file tree
Hide file tree
Showing 421 changed files with 1,916 additions and 2,362 deletions.
118 changes: 47 additions & 71 deletions components/datetime/src/pattern/names.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1997,22 +1997,6 @@ impl<FSet: DateTimeNamesMarker> RawDateTimeNames<FSet> {
Ok(())
}

fn load_mz_periods<P>(
&mut self,
provider: &P,
error_field: ErrorField,
) -> Result<(), PatternLoadError>
where
P: BoundDataProvider<tz::MzPeriodV1Marker> + ?Sized,
{
let variables = ();
self.mz_periods
.load_put(provider, Default::default(), variables)
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?;
Ok(())
}

pub(crate) fn load_time_zone_generic_long_names(
&mut self,
provider: &(impl BoundDataProvider<tz::MzGenericLongV1Marker> + ?Sized),
Expand All @@ -2031,24 +2015,22 @@ impl<FSet: DateTimeNamesMarker> RawDateTimeNames<FSet> {
id: DataIdentifierBorrowed::for_locale(&locale),
..Default::default()
};
self.mz_generic_long
let cs1 = self
.mz_generic_long
.load_put(provider, req, variables)
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?;
self.load_mz_periods(mz_period_provider, error_field)?;
#[allow(clippy::unwrap_used)] // we just loaded them
if self
.mz_generic_long
.get()
.inner
.get_option()
.unwrap()
.checksum
!= self.mz_periods.get().inner.get_option().unwrap().checksum
{
.map_err(|e| PatternLoadError::Data(e, error_field))?
.checksum;
let cs2 = self
.mz_periods
.load_put(mz_period_provider, Default::default(), ())
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?
.checksum;
if cs1.is_none() || cs1 != cs2 {
return Err(PatternLoadError::Data(
DataErrorKind::InconsistentData(tz::MzPeriodV1Marker::INFO)
.with_req(tz::MzGenericLongV1Marker::INFO, req),
.with_req(tz::MzSpecificLongV1Marker::INFO, req),
error_field,
));
}
Expand All @@ -2073,24 +2055,22 @@ impl<FSet: DateTimeNamesMarker> RawDateTimeNames<FSet> {
id: DataIdentifierBorrowed::for_locale(&locale),
..Default::default()
};
self.mz_generic_short
let cs1 = self
.mz_generic_short
.load_put(provider, req, variables)
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?;
self.load_mz_periods(mz_period_provider, error_field)?;
#[allow(clippy::unwrap_used)] // we just loaded them
if self
.mz_generic_short
.get()
.inner
.get_option()
.unwrap()
.checksum
!= self.mz_periods.get().inner.get_option().unwrap().checksum
{
.map_err(|e| PatternLoadError::Data(e, error_field))?
.checksum;
let cs2 = self
.mz_periods
.load_put(mz_period_provider, Default::default(), ())
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?
.checksum;
if cs1.is_none() || cs1 != cs2 {
return Err(PatternLoadError::Data(
DataErrorKind::InconsistentData(tz::MzPeriodV1Marker::INFO)
.with_req(tz::MzGenericShortV1Marker::INFO, req),
.with_req(tz::MzSpecificLongV1Marker::INFO, req),
error_field,
));
}
Expand All @@ -2115,21 +2095,19 @@ impl<FSet: DateTimeNamesMarker> RawDateTimeNames<FSet> {
id: DataIdentifierBorrowed::for_locale(&locale),
..Default::default()
};
self.mz_specific_long
let cs1 = self
.mz_specific_long
.load_put(provider, req, variables)
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?;
self.load_mz_periods(mz_period_provider, error_field)?;
#[allow(clippy::unwrap_used)] // we just loaded them
if self
.mz_specific_long
.get()
.inner
.get_option()
.unwrap()
.checksum
!= self.mz_periods.get().inner.get_option().unwrap().checksum
{
.map_err(|e| PatternLoadError::Data(e, error_field))?
.checksum;
let cs2 = self
.mz_periods
.load_put(mz_period_provider, Default::default(), ())
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?
.checksum;
if cs1.is_none() || cs1 != cs2 {
return Err(PatternLoadError::Data(
DataErrorKind::InconsistentData(tz::MzPeriodV1Marker::INFO)
.with_req(tz::MzSpecificLongV1Marker::INFO, req),
Expand Down Expand Up @@ -2157,24 +2135,22 @@ impl<FSet: DateTimeNamesMarker> RawDateTimeNames<FSet> {
id: DataIdentifierBorrowed::for_locale(&locale),
..Default::default()
};
self.mz_specific_short
let cs1 = self
.mz_specific_short
.load_put(provider, req, variables)
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?;
self.load_mz_periods(mz_period_provider, error_field)?;
#[allow(clippy::unwrap_used)] // we just loaded them
if self
.mz_specific_short
.get()
.inner
.get_option()
.unwrap()
.checksum
!= self.mz_periods.get().inner.get_option().unwrap().checksum
{
.map_err(|e| PatternLoadError::Data(e, error_field))?
.checksum;
let cs2 = self
.mz_periods
.load_put(mz_period_provider, Default::default(), ())
.map_err(|e| MaybePayloadError::into_load_error(e, error_field))?
.map_err(|e| PatternLoadError::Data(e, error_field))?
.checksum;
if cs1.is_none() || cs1 != cs2 {
return Err(PatternLoadError::Data(
DataErrorKind::InconsistentData(tz::MzPeriodV1Marker::INFO)
.with_req(tz::MzSpecificShortV1Marker::INFO, req),
.with_req(tz::MzSpecificLongV1Marker::INFO, req),
error_field,
));
}
Expand Down
18 changes: 6 additions & 12 deletions components/datetime/src/provider/time_zones.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ pub struct ExemplarCitiesV1<'data> {
/// An ICU4X mapping to generic metazone names.
/// See CLDR-JSON timeZoneNames.json for more context.
///
/// These markers use a checksum to ensure consistency with [`MetazonePeriodV1`].
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
Expand All @@ -168,11 +170,6 @@ pub struct ExemplarCitiesV1<'data> {
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
#[yoke(prove_covariance_manually)]
pub struct MetazoneGenericNamesV1<'data> {
/// An XxHash64 checksum of the full metazone names.
///
/// The checksum here should match the checksum in [`MetazonePeriodV1`]
/// if these were generated from the same data set.
pub checksum: u64,
/// The default mapping between metazone id and localized metazone name.
#[cfg_attr(feature = "serde", serde(borrow))]
pub defaults: ZeroMap<'data, MetazoneId, str>,
Expand All @@ -185,6 +182,8 @@ pub struct MetazoneGenericNamesV1<'data> {
/// Specific names include time variants such as "daylight."
/// See CLDR-JSON timeZoneNames.json for more context.
///
/// These markers use a checksum to ensure consistency with [`MetazonePeriodV1`].
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
Expand All @@ -200,11 +199,6 @@ pub struct MetazoneGenericNamesV1<'data> {
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
#[yoke(prove_covariance_manually)]
pub struct MetazoneSpecificNamesV1<'data> {
/// An XxHash64 checksum of the full metazone names.
///
/// The checksum here should match the checksum in [`MetazonePeriodV1`]
/// if these were generated from the same data set.
pub checksum: u64,
/// The default mapping between metazone id and localized metazone name.
#[cfg_attr(feature = "serde", serde(borrow))]
pub defaults: ZeroMap<'data, (MetazoneId, ZoneVariant), str>,
Expand All @@ -225,6 +219,8 @@ pub type MetazoneId = core::num::NonZeroU8;
/// An ICU4X mapping to the metazones at a given period.
/// See CLDR-JSON metaZones.json for more context.
///
/// This markers uses a checksum to ensure consistency with [`Metazone*NamesV1`].
///
/// <div class="stab unstable">
/// 🚧 This code is considered unstable; it may change at any time, in breaking or non-breaking ways,
/// including in SemVer minor releases. While the serde representation of data structs is guaranteed
Expand All @@ -245,6 +241,4 @@ pub struct MetazonePeriodV1<'data> {
/// the number of minutes since the local [`EPOCH`](icu_timezone::provider::EPOCH). It represents when the metazone started to be used.
#[cfg_attr(feature = "serde", serde(borrow))]
pub list: ZeroMap2d<'data, TimeZoneBcp47Id, IsoMinutesSinceEpoch, NichedOption<MetazoneId, 1>>,
/// An XxHash64 checksum of the full metazone names.
pub checksum: u64,
}
10 changes: 5 additions & 5 deletions components/datetime/src/scaffold/names_storage.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ pub trait MaybePayload<M: DynamicDataMarker, Variables>: UnstableSealed {
provider: &P,
req: DataRequest,
variables: Variables,
) -> Result<Result<(), DataError>, MaybePayloadError>
) -> Result<Result<DataResponseMetadata, DataError>, MaybePayloadError>
where
P: BoundDataProvider<M> + ?Sized,
Self: Sized;
Expand Down Expand Up @@ -199,15 +199,15 @@ where
provider: &P,
req: DataRequest,
variables: Variables,
) -> Result<Result<(), DataError>, MaybePayloadError>
) -> Result<Result<DataResponseMetadata, DataError>, MaybePayloadError>
where
P: BoundDataProvider<M> + ?Sized,
Self: Sized,
{
let arg_variables = variables;
match &self.inner {
OptionalNames::SingleLength { variables, .. } if arg_variables == *variables => {
return Ok(Ok(()));
return Ok(Ok(Default::default()));
}
OptionalNames::SingleLength { .. } => {
return Err(MaybePayloadError::ConflictingField);
Expand All @@ -220,7 +220,7 @@ where
payload: response.payload,
variables: arg_variables,
};
Ok(Ok(()))
Ok(Ok(response.metadata))
}
Err(e) => Ok(Err(e)),
}
Expand All @@ -242,7 +242,7 @@ impl<M: DynamicDataMarker, Variables> MaybePayload<M, Variables> for () {
_: &P,
_: DataRequest,
_: Variables,
) -> Result<Result<(), DataError>, MaybePayloadError>
) -> Result<Result<DataResponseMetadata, DataError>, MaybePayloadError>
where
P: BoundDataProvider<M> + ?Sized,
Self: Sized,
Expand Down
49 changes: 31 additions & 18 deletions components/timezone/src/ids.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ use crate::{
#[derive(Debug, Clone)]
pub struct TimeZoneIdMapper {
data: DataPayload<IanaToBcp47MapV3Marker>,
checksum: u64,
}

impl TimeZoneIdMapper {
Expand Down Expand Up @@ -132,8 +133,14 @@ impl TimeZoneIdMapper {
where
P: DataProvider<IanaToBcp47MapV3Marker> + ?Sized,
{
let data = provider.load(Default::default())?.payload;
Ok(Self { data })
let response = provider.load(Default::default())?;
Ok(Self {
data: response.payload,
checksum: response.metadata.checksum.ok_or_else(|| {
DataError::custom("Missing checksum")
.with_req(IanaToBcp47MapV3Marker::INFO, Default::default())
})?,
})
}

/// Returns a borrowed version of the mapper that can be queried.
Expand All @@ -142,6 +149,7 @@ impl TimeZoneIdMapper {
pub fn as_borrowed(&self) -> TimeZoneIdMapperBorrowed {
TimeZoneIdMapperBorrowed {
data: self.data.get(),
checksum: self.checksum,
}
}
}
Expand All @@ -158,6 +166,7 @@ impl AsRef<TimeZoneIdMapper> for TimeZoneIdMapper {
#[derive(Debug, Copy, Clone)]
pub struct TimeZoneIdMapperBorrowed<'a> {
data: &'a IanaToBcp47MapV3<'a>,
checksum: u64,
}

#[cfg(feature = "compiled_data")]
Expand All @@ -179,6 +188,7 @@ impl TimeZoneIdMapperBorrowed<'static> {
pub fn new() -> Self {
Self {
data: crate::provider::Baked::SINGLETON_IANA_TO_BCP47_MAP_V3_MARKER,
checksum: crate::provider::Baked::SINGLETON_IANA_TO_BCP47_MAP_V3_MARKER_CHECKSUM,
}
}

Expand All @@ -189,6 +199,7 @@ impl TimeZoneIdMapperBorrowed<'static> {
pub fn static_to_owned(&self) -> TimeZoneIdMapper {
TimeZoneIdMapper {
data: DataPayload::from_static_ref(self.data),
checksum: self.checksum,
}
}
}
Expand Down Expand Up @@ -597,13 +608,20 @@ where
/// [📚 Help choosing a constructor](icu_provider::constructors)
#[cfg(feature = "compiled_data")]
pub fn try_new_with_mapper(mapper: I) -> Result<Self, DataError> {
Self {
if mapper.as_ref().checksum
!= crate::provider::Baked::SINGLETON_BCP47_TO_IANA_MAP_V1_MARKER_CHECKSUM
{
return Err(
DataErrorKind::InconsistentData(IanaToBcp47MapV3Marker::INFO)
.with_marker(Bcp47ToIanaMapV1Marker::INFO),
);
}
Ok(Self {
inner: mapper,
data: DataPayload::from_static_ref(
crate::provider::Baked::SINGLETON_BCP47_TO_IANA_MAP_V1_MARKER,
),
}
.validated()
})
}

icu_provider::gen_any_buffer_data_constructors!((mapper: I) -> error: DataError,
Expand All @@ -621,22 +639,17 @@ where
where
P: DataProvider<IanaToBcp47MapV3Marker> + DataProvider<Bcp47ToIanaMapV1Marker> + ?Sized,
{
let data = provider.load(Default::default())?.payload;
Self {
inner: mapper,
data,
}
.validated()
}

fn validated(self) -> Result<Self, DataError> {
if self.inner.as_ref().data.get().bcp47_ids_checksum != self.data.get().bcp47_ids_checksum {
let response = provider.load(Default::default())?;
if Some(mapper.as_ref().checksum) != response.metadata.checksum {
return Err(
DataErrorKind::InconsistentData(IanaToBcp47MapV3Marker::INFO)
.with_marker(Bcp47ToIanaMapV1Marker::INFO),
);
}
Ok(self)
Ok(Self {
inner: mapper,
data: response.payload,
})
}

/// Gets the inner [`TimeZoneIdMapper`] for performing queries.
Expand Down Expand Up @@ -681,8 +694,8 @@ impl TimeZoneIdMapperWithFastCanonicalizationBorrowed<'static> {
#[cfg(feature = "compiled_data")]
pub fn new() -> Self {
const _: () = assert!(
crate::provider::Baked::SINGLETON_IANA_TO_BCP47_MAP_V3_MARKER.bcp47_ids_checksum
== crate::provider::Baked::SINGLETON_BCP47_TO_IANA_MAP_V1_MARKER.bcp47_ids_checksum,
crate::provider::Baked::SINGLETON_IANA_TO_BCP47_MAP_V3_MARKER_CHECKSUM
== crate::provider::Baked::SINGLETON_BCP47_TO_IANA_MAP_V1_MARKER_CHECKSUM,
);
Self {
inner: TimeZoneIdMapperBorrowed::new(),
Expand Down
Loading

0 comments on commit 009cf1a

Please sign in to comment.