From ba3af4686d87c7bcd2ca8d7634a87a3ca1366de3 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 17:00:47 +0800 Subject: [PATCH 01/39] Rename key! to vakey! --- crates/macro/src/lib.rs | 28 +++++++++++++++++++++++++++- src/lib.rs | 4 ++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/macro/src/lib.rs b/crates/macro/src/lib.rs index 30d23ff..e71a832 100644 --- a/crates/macro/src/lib.rs +++ b/crates/macro/src/lib.rs @@ -264,8 +264,34 @@ fn generate_code( } } +/// A procedural macro that generates a string representation of the input. +/// +/// This macro accepts either a string literal or an identifier as input. +/// If the input is a string literal, it returns the value of the string literal. +/// If the input is an identifier, it returns the string representation of the identifier. +/// +/// # Arguments +/// +/// * `input` - The input token stream. It should be either a string literal or an identifier. +/// +/// # Returns +/// +/// Returns a token stream that contains a string representation of the input. If the input cannot be parsed as a string literal or an identifier, +/// it returns a compile error. +/// +/// # Example +/// +/// ```no_run +/// # use rust_i18n::vakey; +/// # fn v1() { +/// let key = vakey!(name); +/// # } +/// # fn v2() { +/// let key = vakey!("name"); +/// # } +/// ``` #[proc_macro] -pub fn key(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +pub fn vakey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let output = syn::parse::(input.clone()) .map(|str| str.value()) .or(syn::parse::(input.clone()).map(|ident| format!("{}", ident))); diff --git a/src/lib.rs b/src/lib.rs index 59ab8c2..9ed8a14 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,7 @@ use once_cell::sync::Lazy; #[doc(hidden)] pub use once_cell; -pub use rust_i18n_macro::{i18n, key}; +pub use rust_i18n_macro::{i18n, vakey}; pub use rust_i18n_support::{AtomicStr, Backend, BackendExt, SimpleBackend}; static CURRENT_LOCALE: Lazy = Lazy::new(|| AtomicStr::from("en")); @@ -124,7 +124,7 @@ macro_rules! t { { let message = crate::_rust_i18n_translate($locale, $key); let patterns: &[&str] = &[ - $(rust_i18n::key!($var_name)),+ + $(rust_i18n::vakey!($var_name)),+ ]; let values = &[ $(format!("{}", $var_val)),+ From 903c3d1bf7dfded446d8e157a8478a2bc4f5b597 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 17:05:58 +0800 Subject: [PATCH 02/39] Add crate::_rust_i18n_try_translate() to returns None on untranslated --- crates/macro/src/lib.rs | 44 ++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/crates/macro/src/lib.rs b/crates/macro/src/lib.rs index e71a832..59520c4 100644 --- a/crates/macro/src/lib.rs +++ b/crates/macro/src/lib.rs @@ -229,30 +229,34 @@ fn generate_code( #[inline] #[allow(missing_docs)] pub fn _rust_i18n_translate<'r>(locale: &str, key: &'r str) -> Cow<'r, str> { - if let Some(value) = _RUST_I18N_BACKEND.translate(locale, key) { - return value.into(); - } - - let mut current_locale = locale; - while let Some(fallback_locale) = _rust_i18n_lookup_fallback(current_locale) { - if let Some(value) = _RUST_I18N_BACKEND.translate(fallback_locale, key) { - return value.into(); + _rust_i18n_try_translate(locale, key).unwrap_or_else(|| { + if locale.is_empty() { + key.into() + } else { + format!("{}.{}", locale, key).into() } - current_locale = fallback_locale; - } + }) + } - if let Some(fallback) = _RUST_I18N_FALLBACK_LOCALE { - for locale in fallback { - if let Some(value) = _RUST_I18N_BACKEND.translate(locale, key) { - return value.into(); + /// Try to get I18n text by locale and key + #[inline] + #[allow(missing_docs)] + pub fn _rust_i18n_try_translate<'r>(locale: &str, key: impl AsRef) -> Option> { + _RUST_I18N_BACKEND.translate(locale, key.as_ref()) + .map(Cow::from) + .or_else(|| { + let mut current_locale = locale; + while let Some(fallback_locale) = _rust_i18n_lookup_fallback(current_locale) { + if let Some(value) = _RUST_I18N_BACKEND.translate(fallback_locale, key.as_ref()) { + return Some(Cow::from(value)); + } + current_locale = fallback_locale; } - } - } - if locale.is_empty() { - return key.into(); - } - return format!("{}.{}", locale, key).into(); + _RUST_I18N_FALLBACK_LOCALE.and_then(|fallback| { + fallback.iter().find_map(|locale| _RUST_I18N_BACKEND.translate(locale, key.as_ref()).map(Cow::from)) + }) + }) } #[allow(missing_docs)] From 99ff77bd4631a79075d530ed0ccc6faeaeadb8ee Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 17:11:25 +0800 Subject: [PATCH 03/39] Add CowStr<'a> and TrKey for new tr! --- crates/support/Cargo.toml | 2 + crates/support/src/cow_str.rs | 100 ++++++++++++++++++++++++++++++++ crates/support/src/lib.rs | 4 ++ crates/support/src/tr_key.rs | 106 ++++++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 crates/support/src/cow_str.rs create mode 100644 crates/support/src/tr_key.rs diff --git a/crates/support/Cargo.toml b/crates/support/Cargo.toml index ff82f15..eab6f28 100644 --- a/crates/support/Cargo.toml +++ b/crates/support/Cargo.toml @@ -9,12 +9,14 @@ version = "3.0.1" [dependencies] arc-swap = "1.6.0" +base62 = "2.0.2" globwalk = "0.8.1" once_cell = "1.10.0" proc-macro2 = "1.0" serde = "1" serde_json = "1" serde_yaml = "0.8" +siphasher = "1.0.0" toml = "0.7.4" normpath = "1.1.1" lazy_static = "1" diff --git a/crates/support/src/cow_str.rs b/crates/support/src/cow_str.rs new file mode 100644 index 0000000..2544807 --- /dev/null +++ b/crates/support/src/cow_str.rs @@ -0,0 +1,100 @@ +use std::borrow::Cow; +use std::sync::Arc; + +/// A wrapper for `Cow<'a, str>` that is specifically designed for use with the `tr!` macro. +/// +/// This wrapper provides additional functionality or optimizations when handling strings in the `tr!` macro. +pub struct CowStr<'a>(Cow<'a, str>); + +impl<'a> CowStr<'a> { + pub fn into_inner(self) -> Cow<'a, str> { + self.0 + } +} + +macro_rules! impl_convert_from_numeric { + ($typ:ty) => { + impl<'a> From<$typ> for CowStr<'a> { + fn from(val: $typ) -> Self { + Self(Cow::from(format!("{}", val))) + } + } + }; +} + +impl_convert_from_numeric!(i8); +impl_convert_from_numeric!(i16); +impl_convert_from_numeric!(i32); +impl_convert_from_numeric!(i64); +impl_convert_from_numeric!(i128); +impl_convert_from_numeric!(isize); + +impl_convert_from_numeric!(u8); +impl_convert_from_numeric!(u16); +impl_convert_from_numeric!(u32); +impl_convert_from_numeric!(u64); +impl_convert_from_numeric!(u128); +impl_convert_from_numeric!(usize); + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Arc) -> Self { + Self(Cow::Owned(s.to_string())) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Box) -> Self { + Self(Cow::Owned(s.to_string())) + } +} + +impl<'a> From<&'a str> for CowStr<'a> { + #[inline] + fn from(s: &'a str) -> Self { + Self(Cow::Borrowed(s)) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Arc<&'a str>) -> Self { + Self(Cow::Borrowed(*s)) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Box<&'a str>) -> Self { + Self(Cow::Borrowed(*s)) + } +} + +impl<'a> From for CowStr<'a> { + #[inline] + fn from(s: String) -> Self { + Self(Cow::from(s)) + } +} + +impl<'a> From<&'a String> for CowStr<'a> { + #[inline] + fn from(s: &'a String) -> Self { + Self(Cow::Borrowed(s)) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Arc) -> Self { + Self(Cow::Owned(s.to_string())) + } +} + +impl<'a> From> for CowStr<'a> { + #[inline] + fn from(s: Box) -> Self { + Self(Cow::from(*s)) + } +} diff --git a/crates/support/src/lib.rs b/crates/support/src/lib.rs index a8fadd1..7663a5a 100644 --- a/crates/support/src/lib.rs +++ b/crates/support/src/lib.rs @@ -5,8 +5,12 @@ use std::{collections::HashMap, path::Path}; mod atomic_str; mod backend; +mod cow_str; +mod tr_key; pub use atomic_str::AtomicStr; pub use backend::{Backend, BackendExt, SimpleBackend}; +pub use cow_str::CowStr; +pub use tr_key::{TrKey, TrKeyNumeric, TR_KEY_PREFIX}; type Locale = String; type Value = serde_json::Value; diff --git a/crates/support/src/tr_key.rs b/crates/support/src/tr_key.rs new file mode 100644 index 0000000..6278677 --- /dev/null +++ b/crates/support/src/tr_key.rs @@ -0,0 +1,106 @@ +use once_cell::sync::Lazy; +use siphasher::sip128::SipHasher13; + +/// The prefix of auto-generated literal translation key +pub const TR_KEY_PREFIX: &str = "tr_"; + +// The hasher for generate the literal translation key +static TR_KEY_HASHER: Lazy = Lazy::new(SipHasher13::new); + +pub trait TrKeyNumeric: std::fmt::Display { + fn tr_key_numeric(&self) -> String { + format!("{}N_{}", TR_KEY_PREFIX, self) + } +} + +/// A trait for generating translation key from a value. +pub trait TrKey { + fn tr_key(&self) -> String; +} + +macro_rules! impl_tr_key_for_numeric { + ($typ:ty) => { + impl TrKeyNumeric for $typ {} + impl TrKey for $typ { + #[inline] + fn tr_key(&self) -> String { + self.tr_key_numeric() + } + } + }; +} + +macro_rules! impl_tr_key_for_signed_numeric { + ($typ:ty) => { + impl TrKeyNumeric for $typ {} + impl TrKey for $typ { + #[inline] + fn tr_key(&self) -> String { + (*self as u128).tr_key_numeric() + } + } + }; +} + +impl_tr_key_for_numeric!(u8); +impl_tr_key_for_numeric!(u16); +impl_tr_key_for_numeric!(u32); +impl_tr_key_for_numeric!(u64); +impl_tr_key_for_numeric!(u128); +impl_tr_key_for_numeric!(usize); +impl_tr_key_for_signed_numeric!(i8); +impl_tr_key_for_signed_numeric!(i16); +impl_tr_key_for_signed_numeric!(i32); +impl_tr_key_for_signed_numeric!(i64); +impl_tr_key_for_signed_numeric!(i128); +impl_tr_key_for_signed_numeric!(isize); + +impl TrKey for [u8] { + #[inline] + fn tr_key(&self) -> String { + let hash = TR_KEY_HASHER.hash(self).as_u128(); + format!("{}{}", TR_KEY_PREFIX, base62::encode(hash)) + } +} + +impl TrKey for str { + #[inline] + fn tr_key(&self) -> String { + self.as_bytes().tr_key() + } +} + +impl TrKey for &str { + #[inline] + fn tr_key(&self) -> String { + self.as_bytes().tr_key() + } +} + +impl TrKey for String { + #[inline] + fn tr_key(&self) -> String { + self.as_bytes().tr_key() + } +} + +impl TrKey for &String { + #[inline] + fn tr_key(&self) -> String { + self.as_bytes().tr_key() + } +} + +impl<'a> TrKey for std::borrow::Cow<'a, str> { + #[inline] + fn tr_key(&self) -> String { + self.as_bytes().tr_key() + } +} + +impl<'a> TrKey for &std::borrow::Cow<'a, str> { + #[inline] + fn tr_key(&self) -> String { + self.as_bytes().tr_key() + } +} From 53b53b1ae94df9f56bcdebe0c6463c7be94c7cb5 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 17:29:02 +0800 Subject: [PATCH 04/39] Add new `tr!` macro to get translations with string literals supports --- Cargo.toml | 3 + crates/macro/Cargo.toml | 3 + crates/macro/src/lib.rs | 63 +++++++- crates/macro/src/tr.rs | 331 ++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 +- 5 files changed, 401 insertions(+), 3 deletions(-) create mode 100644 crates/macro/src/tr.rs diff --git a/Cargo.toml b/Cargo.toml index 5f5abd7..0f05101 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,3 +45,6 @@ members = [ [[bench]] harness = false name = "bench" + +[features] +log_tr_dyn = ["rust-i18n-macro/log_tr_dyn"] diff --git a/crates/macro/Cargo.toml b/crates/macro/Cargo.toml index 0050a4c..1c8ed5a 100644 --- a/crates/macro/Cargo.toml +++ b/crates/macro/Cargo.toml @@ -25,3 +25,6 @@ rust-i18n = { path = "../.." } [lib] proc-macro = true + +[features] +log_tr_dyn = [] diff --git a/crates/macro/src/lib.rs b/crates/macro/src/lib.rs index 59520c4..90fdbc0 100644 --- a/crates/macro/src/lib.rs +++ b/crates/macro/src/lib.rs @@ -3,6 +3,8 @@ use rust_i18n_support::{is_debug, load_locales}; use std::collections::HashMap; use syn::{parse_macro_input, Expr, Ident, LitStr, Token}; +mod tr; + struct Args { locales_path: String, fallback: Option>, @@ -197,7 +199,7 @@ fn generate_code( // result quote! { - use rust_i18n::BackendExt; + use rust_i18n::{BackendExt, CowStr, TrKey}; use std::borrow::Cow; /// I18n backend instance @@ -305,3 +307,62 @@ pub fn vakey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { Err(err) => err.to_compile_error().into(), } } + +/// Get I18n text with literals supports. +/// +/// This macro first checks if a translation exists for the input string. +/// If it does, it returns the translated string. +/// If it does not, it returns the input string literal. +/// +/// # Variants +/// +/// This macro has several variants that allow for different use cases: +/// +/// * `tr!("foo")`: +/// Translates the string "foo" using the current locale. +/// +/// * `tr!("foo", locale = "en")`: +/// Translates the string "foo" using the specified locale "en". +/// +/// * `tr!("foo", locale = "en", a = 1, b = "Foo")`: +/// Translates the string "foo" using the specified locale "en" and replaces the patterns "{a}" and "{b}" in the string with "1" and "Foo" respectively. +/// +/// * `tr!("foo %{a} %{b}", a = "bar", b = "baz")`: +/// Translates the string "foo %{a} %{b}" using the current locale and replaces the patterns "{a}" and "{b}" in the string with "bar" and "baz" respectively. +/// +/// * `tr!("foo %{a} %{b}", locale = "en", "a" => "bar", "b" => "baz")`: +/// Translates the string "foo %{a} %{b}" using the specified locale "en" and replaces the patterns "{a}" and "{b}" in the string with "bar" and "baz" respectively. +/// +/// * `tr!("foo %{a} %{b}", "a" => "bar", "b" => "baz")`: +/// Translates the string "foo %{a} %{b}" using the current locale and replaces the patterns "{a}" and "{b}" in the string with "bar" and "baz" respectively. +/// +/// # Examples +/// +/// ```no_run +/// #[macro_use] extern crate rust_i18n; +/// # use rust_i18n::{tr, CowStr}; +/// # fn _rust_i18n_try_translate<'r>(locale: &str, key: &'r str) -> Option> { todo!() } +/// # fn main() { +/// // Simple get text with current locale +/// tr!("Hello world"); +/// // => "Hello world" (Key `tr_3RnEdpgZvZ2WscJuSlQJkJ` for "Hello world") +/// +/// // Get a special locale's text +/// tr!("Hello world", locale = "de"); +/// // => "Hallo Welt!" (Key `tr_3RnEdpgZvZ2WscJuSlQJkJ` for "Hello world") +/// +/// // With variables +/// tr!("Hello, %{name}", name = "world"); +/// // => "Hello, world" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hello, %{name}") +/// tr!("Hello, %{name} and %{other}", name = "Foo", other ="Bar"); +/// // => "Hello, Foo and Bar" (Key `tr_3eULVGYoyiBuaM27F93Mo7` for "Hello, %{name} and %{other}") +/// +/// // With locale and variables +/// tr!("Hallo, %{name}", locale = "de", name = "Jason"); +/// // => "Hallo, Jason" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hallo, %{name}") +/// # } +/// ``` +#[proc_macro] +pub fn tr(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + parse_macro_input!(input as tr::Tr).into() +} diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs new file mode 100644 index 0000000..e22c68d --- /dev/null +++ b/crates/macro/src/tr.rs @@ -0,0 +1,331 @@ +use quote::{quote, ToTokens}; +use rust_i18n_support::TrKey; +use syn::{parse::discouraged::Speculative, Expr, ExprMacro, Ident, LitStr, Token}; + +pub struct Argument { + pub name: String, + pub value: Expr, +} + +impl Argument { + #[allow(dead_code)] + pub fn value_string(&self) -> String { + match &self.value { + Expr::Lit(expr_lit) => match &expr_lit.lit { + syn::Lit::Str(lit_str) => lit_str.value(), + _ => self.value.to_token_stream().to_string(), + }, + _ => self.value.to_token_stream().to_string(), + } + } +} + +impl syn::parse::Parse for Argument { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let name = input + .parse::() + .map(|v| v.to_string()) + .or_else(|_| input.parse::().map(|v| v.value())) + .map_err(|_| input.error("Expected a `string` literal or an identifier"))?; + let _ = input.parse::()?; + let _ = input.parse::]>>()?; + let value = input.parse::()?; + Ok(Self { name, value }) + } +} + +#[derive(Default)] +pub struct Arguments { + pub args: Vec, +} + +impl Arguments { + pub fn is_empty(&self) -> bool { + self.args.is_empty() + } + + pub fn keys(&self) -> Vec { + self.args.iter().map(|arg| arg.name.clone()).collect() + } + + pub fn values(&self) -> Vec { + self.args.iter().map(|arg| arg.value.clone()).collect() + } +} + +impl AsRef> for Arguments { + fn as_ref(&self) -> &Vec { + &self.args + } +} + +impl AsMut> for Arguments { + fn as_mut(&mut self) -> &mut Vec { + &mut self.args + } +} + +impl syn::parse::Parse for Arguments { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let args = input + .parse_terminated(Argument::parse, Token![,])? + .into_iter() + .collect(); + Ok(Self { args }) + } +} + +#[derive(Default)] +pub enum Messagekind { + #[default] + Literal, + Expr, + ExprCall, + ExprClosure, + ExprMacro, + ExprReference, + ExprUnary, + Ident, +} + +#[derive(Default)] +pub struct Messsage { + pub key: proc_macro2::TokenStream, + pub val: proc_macro2::TokenStream, + pub kind: Messagekind, +} + +impl Messsage { + fn try_exp(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let expr = fork.parse::()?; + let key = quote! { #expr }; + let val = quote! { #expr }; + input.advance_to(&fork); + let kind = match expr { + Expr::Call(_) => Messagekind::ExprCall, + Expr::Closure(_) => Messagekind::ExprClosure, + Expr::Macro(_) => Messagekind::ExprMacro, + Expr::Reference(_) => Messagekind::ExprReference, + Expr::Unary(_) => Messagekind::ExprUnary, + _ => Messagekind::Expr, + }; + Ok(Self { key, val, kind }) + } + + #[allow(dead_code)] + fn try_exp_macro(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let expr = fork.parse::()?; + let key = quote! { #expr }; + let val = quote! { #expr }; + input.advance_to(&fork); + Ok(Self { + key, + val, + kind: Messagekind::ExprMacro, + }) + } + + fn try_ident(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let ident = fork.parse::()?; + let key = quote! { #ident }; + let val = quote! { #ident }; + input.advance_to(&fork); + Ok(Self { + key, + val, + kind: Messagekind::Ident, + }) + } + + fn try_litreal(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let lit_str = fork.parse::()?; + let key = lit_str.value().tr_key(); + let key = quote! { #key }; + let val = quote! { #lit_str }; + input.advance_to(&fork); + Ok(Self { + key, + val, + kind: Messagekind::Literal, + }) + } +} + +impl syn::parse::Parse for Messsage { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let result = Self::try_litreal(input) + // .or_else(|_| Self::try_exp_macro(input)) + .or_else(|_| Self::try_exp(input)) + .or_else(|_| Self::try_ident(input))?; + Ok(result) + } +} + +/// A type representing the `tr!` macro. +#[derive(Default)] +pub(crate) struct Tr { + pub msg: Messsage, + pub args: Arguments, + pub locale: Option, +} + +impl Tr { + fn into_token_stream(self) -> proc_macro2::TokenStream { + let msg_key = self.msg.key; + let msg_val = self.msg.val; + let msg_kind = self.msg.kind; + let locale = self.locale.map_or_else( + || quote! { rust_i18n::locale().as_str() }, + |locale| quote! { #locale }, + ); + let keys: Vec<_> = self.args.keys().iter().map(|v| quote! { #v }).collect(); + let values: Vec<_> = self + .args + .values() + .iter() + .map(|v| quote! { format!("{}", #v) }) + .collect(); + match msg_kind { + Messagekind::Literal => { + if self.args.is_empty() { + quote! { + crate::_rust_i18n_try_translate(#locale, #msg_key).unwrap_or_else(|| std::borrow::Cow::from(#msg_val)) + } + } else { + quote! { + { + let msg_key = #msg_key; + let msg_val = #msg_val; + let keys = &[#(#keys),*]; + let values = &[#(#values),*]; + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { + let replaced = rust_i18n::replace_patterns(&translated, keys, values); + std::borrow::Cow::from(replaced) + } else { + let replaced = rust_i18n::replace_patterns(&rust_i18n::CowStr::from(msg_val).into_inner(), keys, values); + std::borrow::Cow::from(replaced) + } + } + } + } + } + Messagekind::ExprCall | Messagekind::ExprClosure | Messagekind::ExprMacro => { + let logging = if cfg!(feature = "log_tr_dyn") { + quote! { + log::debug!("tr: missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); + } + } else { + quote! {} + }; + if self.args.is_empty() { + quote! { + { + let msg_val = #msg_val; + let msg_key = rust_i18n::TrKey::tr_key(&msg_val); + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, msg_key) { + translated + } else { + #logging + std::borrow::Cow::from(msg_val) + } + } + } + } else { + quote! { + { + let msg_val = #msg_val; + let msg_key = rust_i18n::TrKey::tr_key(&msg_val); + let keys = &[#(#keys),*]; + let values = &[#(#values),*]; + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, msg_key) { + let replaced = rust_i18n::replace_patterns(&translated, keys, values); + std::borrow::Cow::from(replaced) + } else { + #logging + let replaced = rust_i18n::replace_patterns(&rust_i18n::CowStr::from(msg_val).into_inner(), keys, values); + std::borrow::Cow::from(replaced) + } + } + } + } + } + Messagekind::Expr + | Messagekind::ExprReference + | Messagekind::ExprUnary + | Messagekind::Ident => { + let logging = if cfg!(feature = "log_tr_dyn") { + quote! { + log::debug!("tr: missing: {} => {:?} @ {}:{}", msg_key, #msg_val, file!(), line!()); + } + } else { + quote! {} + }; + if self.args.is_empty() { + quote! { + { + let msg_key = rust_i18n::TrKey::tr_key(&#msg_key); + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { + translated + } else { + rust_i18n::CowStr::from(#msg_val).into_inner() + } + } + } + } else { + quote! { + { + let msg_key = rust_i18n::TrKey::tr_key(&#msg_key); + let keys = &[#(#keys),*]; + let values = &[#(#values),*]; + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { + let replaced = rust_i18n::replace_patterns(&translated, keys, values); + std::borrow::Cow::from(replaced) + } else { + #logging + let replaced = rust_i18n::replace_patterns(&rust_i18n::CowStr::from(#msg_val).into_inner(), keys, values); + std::borrow::Cow::from(replaced) + } + } + } + } + } + } + } +} + +impl syn::parse::Parse for Tr { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let msg = input.parse::()?; + let comma = input.parse::>()?; + let (args, locale) = if comma.is_some() { + let mut args = input.parse::()?; + let locale = args + .as_ref() + .iter() + .find(|v| v.name == "locale") + .map(|v| v.value.clone()); + args.as_mut().retain(|v| v.name != "locale"); + (args, locale) + } else { + (Arguments::default(), None) + }; + + Ok(Self { msg, args, locale }) + } +} + +impl From for proc_macro::TokenStream { + fn from(args: Tr) -> Self { + args.into_token_stream().into() + } +} + +impl From for proc_macro2::TokenStream { + fn from(args: Tr) -> Self { + args.into_token_stream() + } +} diff --git a/src/lib.rs b/src/lib.rs index 9ed8a14..94edf32 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,8 +6,8 @@ use once_cell::sync::Lazy; #[doc(hidden)] pub use once_cell; -pub use rust_i18n_macro::{i18n, vakey}; -pub use rust_i18n_support::{AtomicStr, Backend, BackendExt, SimpleBackend}; +pub use rust_i18n_macro::{i18n, tr, vakey}; +pub use rust_i18n_support::{AtomicStr, Backend, BackendExt, CowStr, SimpleBackend, TrKey}; static CURRENT_LOCALE: Lazy = Lazy::new(|| AtomicStr::from("en")); From e7e75635812fa03ced7d1640d3fe4532e6e7dba8 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 17:31:12 +0800 Subject: [PATCH 05/39] Add `tr!` support to `rust-i18n-extract` and `rust-i18n-cli` --- crates/cli/src/main.rs | 4 ++-- crates/extract/src/extractor.rs | 27 ++++++++++++++++++--------- crates/extract/src/generator.rs | 17 +++++++++++------ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 6690627..8f94fdf 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -42,8 +42,8 @@ fn main() -> Result<(), Error> { extractor::extract(&mut results, path, source) })?; - let mut messages: Vec<_> = results.values().collect(); - messages.sort_by_key(|m| m.index); + let mut messages: Vec<_> = results.iter().collect(); + messages.sort_by_key(|(_k, m)| m.index); let mut has_error = false; diff --git a/crates/extract/src/extractor.rs b/crates/extract/src/extractor.rs index 613bced..2d1ba55 100644 --- a/crates/extract/src/extractor.rs +++ b/crates/extract/src/extractor.rs @@ -16,20 +16,22 @@ pub struct Location { pub struct Message { pub key: String, pub index: usize, + pub is_tr: bool, pub locations: Vec, } impl Message { - fn new(key: &str, index: usize) -> Self { + fn new(key: &str, index: usize, is_tr: bool) -> Self { Self { key: key.to_owned(), index, + is_tr, locations: vec![], } } } -static METHOD_NAME: &str = "t"; +static METHOD_NAMES: &[&str] = &["t", "tr"]; #[allow(clippy::ptr_arg)] pub fn extract(results: &mut Results, path: &PathBuf, source: &str) -> Result<(), Error> { @@ -63,9 +65,10 @@ impl<'a> Extractor<'a> { } } - if ident == METHOD_NAME && is_macro { + let ident_str = ident.to_string(); + if METHOD_NAMES.contains(&ident_str.as_str()) && is_macro { if let Some(TokenTree::Group(group)) = token_iter.peek() { - self.take_message(group.stream()); + self.take_message(group.stream(), ident_str == "tr"); } } } @@ -76,7 +79,7 @@ impl<'a> Extractor<'a> { Ok(()) } - fn take_message(&mut self, stream: TokenStream) { + fn take_message(&mut self, stream: TokenStream, is_tr: bool) { let mut token_iter = stream.into_iter().peekable(); let literal = if let Some(TokenTree::Literal(literal)) = token_iter.next() { @@ -89,13 +92,18 @@ impl<'a> Extractor<'a> { if let Some(lit) = key { if let Some(key) = literal_to_string(&lit) { - let message_key = format_message_key(&key); - + let (message_key, message_content) = if is_tr { + let hashed_key = rust_i18n_support::TrKey::tr_key(&key); + (hashed_key, key.clone()) + } else { + let message_key = format_message_key(&key); + (message_key.clone(), message_key) + }; let index = self.results.len(); let message = self .results - .entry(message_key.clone()) - .or_insert_with(|| Message::new(&message_key, index)); + .entry(message_key) + .or_insert_with(|| Message::new(&message_content, index, is_tr)); let span = lit.span(); let line = span.start().line; @@ -143,6 +151,7 @@ mod tests { )+ ], index: 0, + is_tr: false, }; results.push(message); )+ diff --git a/crates/extract/src/generator.rs b/crates/extract/src/generator.rs index 4fc2ffc..fa1e253 100644 --- a/crates/extract/src/generator.rs +++ b/crates/extract/src/generator.rs @@ -10,7 +10,7 @@ type Translations = HashMap>; pub fn generate<'a, P: AsRef>( output_path: P, all_locales: &Vec, - messages: impl IntoIterator + Clone, + messages: impl IntoIterator + Clone, ) -> Result<()> { let filename = "TODO.yml"; let format = "yaml"; @@ -63,7 +63,7 @@ fn generate_result<'a, P: AsRef>( output_path: P, output_filename: &str, all_locales: &Vec, - messages: impl IntoIterator + Clone, + messages: impl IntoIterator + Clone, ) -> Translations { let mut trs = Translations::new(); @@ -76,7 +76,7 @@ fn generate_result<'a, P: AsRef>( let ignore_file = |fname: &str| fname.ends_with(&output_filename); let data = load_locales(&output_path, ignore_file); - for m in messages.clone() { + for (key, m) in messages.clone() { if !m.locations.is_empty() { for _l in &m.locations { // TODO: write file and line as YAML comment @@ -85,15 +85,20 @@ fn generate_result<'a, P: AsRef>( } } + let key = if m.is_tr { key } else { &m.key }; if let Some(trs) = data.get(locale) { - if trs.get(&m.key).is_some() { + if trs.get(key).is_some() { continue; } } - let value = m.key.split('.').last().unwrap_or_default(); + let value = if m.is_tr { + m.key.to_owned() + } else { + m.key.split('.').last().unwrap_or_default().to_string() + }; - trs.entry(m.key.clone()) + trs.entry(key.clone()) .or_insert_with(HashMap::new) .insert(locale.to_string(), value.to_string()); } From 2184045b99995127cc85c46bd1102fc5f8d0e8cd Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 17:31:59 +0800 Subject: [PATCH 06/39] Update benchmark and tests for tr! --- benches/bench.rs | 67 ++++++++++++++++++++- tests/integration_tests.rs | 116 ++++++++++++++++++++++++++++++++++++- tests/locales/v2.yml | 81 ++++++++++++++++++++++++++ 3 files changed, 261 insertions(+), 3 deletions(-) diff --git a/benches/bench.rs b/benches/bench.rs index 3e77d47..c85f80b 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -1,4 +1,4 @@ -use rust_i18n::t; +use rust_i18n::{t, tr}; rust_i18n::i18n!("./tests/locales"); @@ -79,6 +79,71 @@ fn bench_t(c: &mut Criterion) { ) }) }); + + c.bench_function("tr", |b| b.iter(|| tr!("hello"))); + + c.bench_function("tr_lorem_ipsum", |b| b.iter(|| tr!( + r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque sed nisi leo. Donec commodo in ex at aliquam. Nunc in aliquam arcu. Fusce mollis metus orci, ut sagittis erat lobortis sed. Morbi quis arcu ultrices turpis finibus tincidunt non in purus. Donec gravida condimentum sapien. Duis iaculis fermentum congue. Quisque blandit libero a lacus auctor vestibulum. Nunc efficitur sollicitudin nisi, sit amet tristique lectus mollis non. Praesent sit amet erat volutpat, pharetra orci eget, rutrum felis. Sed elit augue, imperdiet eu facilisis vel, finibus vel urna. Duis quis neque metus. + + Mauris suscipit bibendum mattis. Vestibulum eu augue diam. Morbi dapibus tempus viverra. Sed aliquam turpis eget justo ornare maximus vitae et tortor. Donec semper neque sit amet sapien congue scelerisque. Maecenas bibendum imperdiet dolor interdum facilisis. Integer non diam tempus, pharetra ex at, euismod diam. Ut enim turpis, sagittis in iaculis ut, finibus et sem. Suspendisse a felis euismod neque euismod placerat. Praesent ipsum libero, porta vel egestas quis, aliquet vitae lorem. Nullam vel pharetra erat, sit amet sodales leo."# + ))); + + c.bench_function("tr_with_locale", |b| b.iter(|| tr!("hello", locale = "en"))); + + c.bench_function("tr_with_threads", |b| { + let exit_loop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mut handles = Vec::new(); + for _ in 0..4 { + let exit_loop = exit_loop.clone(); + handles.push(std::thread::spawn(move || { + while !exit_loop.load(std::sync::atomic::Ordering::SeqCst) { + criterion::black_box(tr!("hello")); + } + })); + } + b.iter(|| tr!("hello")); + exit_loop.store(true, std::sync::atomic::Ordering::SeqCst); + for handle in handles { + handle.join().unwrap(); + } + }); + + c.bench_function("tr_with_args", |b| { + b.iter(|| { + tr!( + "Hello, %{name}. Your message is: %{msg}", + name = "Jason", + msg = "Bla bla" + ) + }) + }); + + c.bench_function("tr_with_args (str)", |b| { + b.iter(|| { + tr!( + "Hello, %{name}. Your message is: %{msg}", + "name" = "Jason", + "msg" = "Bla bla" + ) + }) + }); + + c.bench_function("tr_with_args (many)", |b| { + b.iter(|| { + tr!( + r#"Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. + You live in %{city} %{zip}. + Your website is %{website}."#, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ) + }) + }); } criterion_group!(benches, bench_t); diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 33e8565..879ed06 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -33,7 +33,7 @@ rust_i18n::i18n!( #[cfg(test)] mod tests { - use rust_i18n::t; + use rust_i18n::{t, tr}; use rust_i18n_support::load_locales; mod test0 { @@ -114,7 +114,7 @@ mod tests { fn test_available_locales() { assert_eq!( rust_i18n::available_locales!(), - &["en", "ja", "pt", "zh", "zh-CN"] + &["de", "en", "fr", "ja", "ko", "pt", "ru", "vi", "zh", "zh-CN"] ); } @@ -201,6 +201,118 @@ mod tests { ); } + #[test] + fn test_tr() { + rust_i18n::set_locale("en"); + assert_eq!(tr!("Bar - Hello, World!"), "Bar - Hello, World!"); + + // Vars + assert_eq!( + tr!("Hello, %{name}. Your message is: %{msg}"), + "Hello, %{name}. Your message is: %{msg}" + ); + assert_eq!( + tr!("Hello, %{name}. Your message is: %{msg}", name = "Jason"), + "Hello, Jason. Your message is: %{msg}" + ); + assert_eq!( + tr!( + "Hello, %{name}. Your message is: %{msg}", + name = "Jason", + msg = "Bla bla" + ), + "Hello, Jason. Your message is: Bla bla" + ); + + rust_i18n::set_locale("zh-CN"); + assert_eq!(tr!("Hello, %{name}!", name = "world"), "你好,world!"); + + rust_i18n::set_locale("en"); + assert_eq!(tr!("Hello, %{name}!", name = "world"), "Hello, world!"); + + let fruits = vec!["Apple", "Banana", "Orange"]; + let fruits_translated = vec!["苹果", "香蕉", "橘子"]; + for (src, dst) in fruits.iter().zip(fruits_translated.iter()) { + assert_eq!(tr!(*src, locale = "zh-CN"), *dst); + } + } + + #[test] + fn test_tr_with_tt_val() { + rust_i18n::set_locale("en"); + + assert_eq!( + tr!("You have %{count} messages.", count = 100), + "You have 100 messages." + ); + assert_eq!( + tr!("You have %{count} messages.", count = 1.01), + "You have 1.01 messages." + ); + assert_eq!( + tr!("You have %{count} messages.", count = 1 + 2), + "You have 3 messages." + ); + + // Test end with a comma + assert_eq!( + tr!( + "You have %{count} messages.", + locale = "zh-CN", + count = 1 + 2, + ), + "你收到了 3 条新消息。" + ); + + let a = 100; + assert_eq!( + tr!("You have %{count} messages.", count = a / 2), + "You have 50 messages." + ); + } + + #[test] + fn test_tr_with_locale_and_args() { + rust_i18n::set_locale("en"); + + assert_eq!( + tr!("Bar - Hello, World!", locale = "zh-CN"), + "Bar - 你好世界!" + ); + assert_eq!( + tr!("Bar - Hello, World!", locale = "en"), + "Bar - Hello, World!" + ); + + assert_eq!(tr!("Hello, %{name}!", name = "Jason"), "Hello, Jason!"); + assert_eq!( + tr!("Hello, %{name}!", locale = "en", name = "Jason"), + "Hello, Jason!" + ); + // Invalid locale position, will ignore + assert_eq!( + tr!("Hello, %{name}!", name = "Jason", locale = "en"), + "Hello, Jason!" + ); + assert_eq!( + tr!("Hello, %{name}!", locale = "zh-CN", name = "Jason"), + "你好,Jason!" + ); + } + + #[test] + fn test_tr_with_hash_args() { + rust_i18n::set_locale("en"); + + // Hash args + assert_eq!(tr!("Hello, %{name}!", name => "Jason"), "Hello, Jason!"); + assert_eq!(tr!("Hello, %{name}!", "name" => "Jason"), "Hello, Jason!"); + assert_eq!( + tr!("Hello, %{name}!", locale = "zh-CN", "name" => "Jason"), + "你好,Jason!" + ); + } + #[test] fn test_with_merge_file() { rust_i18n::set_locale("en"); diff --git a/tests/locales/v2.yml b/tests/locales/v2.yml index f32d082..0be5e12 100644 --- a/tests/locales/v2.yml +++ b/tests/locales/v2.yml @@ -11,3 +11,84 @@ nested_locale_test: en: "Hello test3" ja: "こんにちは test3" zh-CN: "你好 test3" +tr_1bHAL18drdyculzJ6OdjT0: + en: "Hello, you id is: 123" + de: "Hallo, deine ID ist: 123" + fr: "Bonjour, votre ID est : 123" + ja: "こんにちは、あなたのIDは 123 です" + ko: "안녕하세요, 당신의 ID는 123입니다" + ru: "Привет, ваш ID: 123" + vi: "Xin chào, ID của bạn là: 123" + zh-CN: "你好,你的 ID 是:123" +tr_29xGXAUPAkgvVzCf9ES3q8: + en: Apple + de: Apfel + fr: Pomme + ja: りんご + ko: 사과 + ru: Яблоко + vi: Táo + zh-CN: 苹果 +tr_2xBwKSjL3poKDdxU7Mej0e: + en: Bar - Hello, World! + de: Bar - Hallo, Welt! + fr: Bar - Bonjour, monde! + ja: Bar - こんにちは世界! + ko: Bar - 안녕하세요, 세계여! + ru: Bar - Привет, мир! + vi: Bar - Xin chào, thế giới! + zh-CN: Bar - 你好世界! +tr_2RohljPx99sA18L8E5oTD4: + en: Orange + de: Orange + fr: Orange + ja: オレンジ + ko: 오렌지 + ru: Апельсин + vi: Cam + zh-CN: 橘子 +tr_7hWbTwvMpr0H0oDLIQlfrm: + en: You have %{count} messages. + de: Du hast %{count} Nachrichten. + fr: Vous avez %{count} messages. + ja: あなたは %{count} 件のメッセージを持っています。 + ko: 당신은 %{count} 개의 메시지를 가지고 있습니다. + ru: У вас %{count} сообщений. + vi: Bạn có %{count} tin nhắn. + zh-CN: 你收到了 %{count} 条新消息。 +tr_7MQVq9vgi0h6pLE47CdWXH: + en: Banana + de: Banane + fr: Banane + ja: バナナ + ko: 바나나 + ru: Банан + vi: Chuối + zh-CN: 香蕉 +tr_kmFrQ2nnJsvUh3Ckxmki0: + en: "Hello, %{name}. Your message is: %{msg}" + de: "Hallo, %{name}. Deine Nachricht ist: %{msg}" + fr: "Bonjour, %{name}. Votre message est: %{msg}" + ja: "こんにちは、%{name}。あなたのメッセージは: %{msg}" + ko: "안녕하세요, %{name}. 당신의 메시지는: %{msg}" + ru: "Привет, %{name}. Ваше сообщение: %{msg}" + vi: "Xin chào, %{name}. Tin nhắn của bạn là: %{msg}" + zh-CN: "你好,%{name}。这是你的消息:%{msg}" +tr_tvpNzQjFwc0trHvgzSBxX: + en: Hello, %{name}! + de: Hallo, %{name}! + fr: Bonjour, %{name}! + ja: こんにちは、%{name}! + ko: 안녕하세요, %{name}! + ru: Привет, %{name}! + vi: Xin chào, %{name}! + zh-CN: 你好,%{name}! +tr_3w1UK0UDyZZMkgsPduhmOv: + en: "Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. \r\n You live in %{city} %{zip}. \r\n Your website is %{website}." + de: "Hallo %{name} %{surname}, deine Kontonummer ist %{id}, E-Mail-Adresse ist %{email}. \r\n Du lebst in %{city} %{zip}. \r\n Deine Website ist %{website}." + fr: "Bonjour %{name} %{surname}, votre identifiant de compte est %{id}, votre adresse e-mail est %{email}. \r\n Vous vivez à %{city} %{zip}. \r\n Votre site Web est %{website}." + ja: "こんにちは %{name} %{surname}、あなたのアカウントIDは %{id}、メールアドレスは %{email} です。 \r\n あなたは %{city} %{zip} に住んでいます。 \r\n あなたのウェブサイトは %{website} です。" + ko: "안녕하세요 %{name} %{surname}, 당신의 계정 ID는 %{id}, 이메일 주소는 %{email} 입니다. \r\n 당신은 %{city} %{zip} 에 살고 있습니다. \r\n 당신의 웹사이트는 %{website} 입니다." + ru: "Привет %{name} %{surname}, ваш ID аккаунта %{id}, адрес электронной почты %{email}. \r\n Вы живете в %{city} %{zip}. \r\n Ваш сайт %{website}." + vi: "Xin chào %{name} %{surname}, ID tài khoản của bạn là %{id}, địa chỉ email là %{email}. \r\n Bạn sống ở %{city} %{zip}. \r\n Trang web của bạn là %{website}." + zh-CN: "你好 %{name} %{surname},你的帐户 ID 是 %{id},电子邮件地址是 %{email}。 \r\n 你住在 %{city} %{zip}。 \r\n 你的网站是 %{website}。" From 479c72c9edc16a7b6dbf7043221f3ba51598e780 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 17:34:38 +0800 Subject: [PATCH 07/39] Add new application example for `tr!` --- Cargo.toml | 1 + examples/app-tr/Cargo.toml | 19 ++++ examples/app-tr/locales/v2.yml | 189 +++++++++++++++++++++++++++++++++ examples/app-tr/src/main.rs | 80 ++++++++++++++ 4 files changed, 289 insertions(+) create mode 100644 examples/app-tr/Cargo.toml create mode 100644 examples/app-tr/locales/v2.yml create mode 100644 examples/app-tr/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 0f05101..55df881 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ members = [ "crates/support", "crates/macro", "examples/app-load-path", + "examples/app-tr", "examples/foo", ] diff --git a/examples/app-tr/Cargo.toml b/examples/app-tr/Cargo.toml new file mode 100644 index 0000000..f991521 --- /dev/null +++ b/examples/app-tr/Cargo.toml @@ -0,0 +1,19 @@ +[package] +edition = "2021" +name = "app-tr" +version = "0.1.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +env_logger = { version = "0.11", optional = true } +log = { version = "0.4", optional = true } +rust-i18n = { path = "../.." } +# rust-i18n-macro = { path = "../../crates/macro" } + +[features] +log_tr_dyn = ["env_logger", "log", "rust-i18n/log_tr_dyn"] + +[package.metadata.i18n] +available-locales = ["en", "zh-CN"] +default-locale = "en" diff --git a/examples/app-tr/locales/v2.yml b/examples/app-tr/locales/v2.yml new file mode 100644 index 0000000..063d422 --- /dev/null +++ b/examples/app-tr/locales/v2.yml @@ -0,0 +1,189 @@ +_version: 2 +tr_29xGXAUPAkgvVzCf9ES3q8: + en: Apple + de: Apfel + fr: Pomme + it: Mela + ja: りんご + ko: 사과 + ru: Яблоко + vi: Táo + zh: 苹果 + zh-TW: 蘋果 +tr_7MQVq9vgi0h6pLE47CdWXH: + en: Banana + de: Banane + fr: Banane + it: Banana + ja: バナナ + ko: 바나나 + ru: Банан + vi: Chuối + zh: 香蕉 + zh-TW: 香蕉 +tr_2RohljPx99sA18L8E5oTD4: + en: Orange + de: Orange + fr: Orange + it: Arancia + ja: オレンジ + ko: 오렌지 + ru: Апельсин + vi: Cam + zh: 橘子 + zh-TW: 橘子 +tr_KtVLkBzuyfHoJZtCWEIOU: + en: Hello + de: Hallo + fr: Bonjour + it: Ciao + ja: こんにちは + ko: 안녕하세요 + ru: Привет + vi: Xin chào + zh: 你好 + zh: 你好 + zh-TW: 妳好 +tr_tvpNzQjFwc0trHvgzSBxX: + en: Hello, %{name}! + de: Hallo, %{name}! + fr: Bonjour, %{name}! + it: Ciao, %{name}! + ja: こんにちは、%{name}! + ko: 안녕하세요, %{name}! + ru: Привет, %{name}! + vi: Xin chào, %{name}! + zh: 你好,%{name}! + zh-TW: 妳好,%{name}! +tr_kmFrQ2nnJsvUh3Ckxmki0: + en: "Hello, %{name}. Your message is: %{msg}" + de: "Hallo, %{name}. Deine Nachricht ist: %{msg}" + fr: "Bonjour, %{name}. Votre message est: %{msg}" + it: "Ciao, %{name}. Il tuo messaggio è: %{msg}" + ja: "こんにちは、%{name}。あなたのメッセージは: %{msg}" + ko: "안녕하세요, %{name}. 당신의 메시지는: %{msg}" + ru: "Привет, %{name}. Ваше сообщение: %{msg}" + vi: "Xin chào, %{name}. Tin nhắn của bạn là: %{msg}" + zh: "你好,%{name}。这是你的消息:%{msg}" + zh-TW: "妳好,%{name}。這是妳的消息:%{msg}" +tr_7hWbTwvMpr0H0oDLIQlfrm: + en: You have %{count} messages. + de: Du hast %{count} Nachrichten. + fr: Vous avez %{count} messages. + it: Hai %{count} messaggi. + ja: あなたは %{count} 件のメッセージを持っています。 + ko: 당신은 %{count} 개의 메시지를 가지고 있습니다. + ru: У вас %{count} сообщений. + vi: Bạn có %{count} tin nhắn. + zh: 你收到了 %{count} 条新消息。 + zh-TW: 妳收到了 %{count} 條新消息。 +tr_N_0: + en: Zero + de: Null + fr: Zéro + it: Zero + ja: ゼロ + ko: 영 + ru: Ноль + vi: Không + zh: 零 + zh-TW: 零 +tr_N_1: + en: One + de: Eins + fr: Un + it: Uno + ja: 一 + ko: 일 + ru: Один + vi: Một + zh: 一 + zh-TW: 一 +tr_N_2: + en: Two + de: Zwei + fr: Deux + it: Due + ja: 二 + ko: 이 + ru: Два + vi: Hai + zh: 二 + zh-TW: 二 +tr_N_3: + en: Three + de: Drei + fr: Trois + it: Tre + ja: 三 + ko: 삼 + ru: Три + vi: Ba + zh: 三 + zh-TW: 三 +tr_N_4: + en: Four + de: Vier + fr: Quatre + it: Quattro + ja: 四 + ko: 사 + ru: Четыре + vi: Bốn + zh: 四 + zh-TW: 四 +tr_N_5: + en: Five + de: Fünf + fr: Cinq + it: Cinque + ja: 五 + ko: 오 + ru: Пять + vi: Năm + zh: 五 + zh-TW: 五 +tr_N_6: + en: Six + de: Sechs + fr: Six + it: Sei + ja: 六 + ko: 육 + ru: Шесть + vi: Sáu + zh: 六 + zh-TW: 六 +tr_N_7: + en: Seven + de: Sieben + fr: Sept + it: Sette + ja: 七 + ko: 칠 + ru: Семь + vi: Bảy + zh: 七 + zh-TW: 七 +tr_N_8: + en: Eight + de: Acht + fr: Huit + it: Otto + ja: 八 + ko: 팔 + ru: Восемь + vi: Tám + zh: 八 + zh-TW: 八 +tr_N_9: + en: Nine + de: Neun + fr: Neuf + it: Nove + ja: 九 + ko: 구 + ru: Девять + vi: Chín + zh: 九 + zh-TW: 九 diff --git a/examples/app-tr/src/main.rs b/examples/app-tr/src/main.rs new file mode 100644 index 0000000..4671206 --- /dev/null +++ b/examples/app-tr/src/main.rs @@ -0,0 +1,80 @@ +use rust_i18n::tr; + +rust_i18n::i18n!("locales"); + +#[cfg(feature = "log_tr_dyn")] +fn set_logger() { + env_logger::builder() + .filter_level(log::LevelFilter::Debug) + .parse_default_env() + .init(); +} + +#[cfg(not(feature = "log_tr_dyn"))] +fn set_logger() {} + +fn main() { + set_logger(); + + let locales = rust_i18n::available_locales!(); + println!("Available locales: {:?}", locales); + println!(); + + println!("String literals with patterns translation:"); + for locale in &locales { + println!( + "{} => {} ({})", + "Hello, %{name}!", + tr!("Hello, %{name}!", name = "World", locale = locale), + locale + ); + } + println!(); + + println!("String literals translation:"); + for locale in &locales { + println!( + "{:>8} => {} ({})", + "Hello", + tr!("Hello", locale = locale), + locale + ); + } + println!(); + + // Identify the locale message by number. + // For example, the `tr!` will find the translation with key named "tr_N_5" for 5. + println!("Numeric literals translation:"); + for locale in &locales { + println!("{:>8} => {} ({})", 5, tr!(5, locale = locale), locale); + } + println!(); + + println!("Missing translations:"); + for locale in &locales { + println!( + "{:>8} => {} ({locale})", + "The message is untranslated!", + tr!("The message is untranslated!", locale = locale) + ); + } + println!(); + + println!("Runtime string translation:"); + let src_list = vec!["Apple", "Banana", "Orange"]; + for src in src_list.iter() { + for locale in &locales { + let translated = tr!(*src, locale = locale); + println!("{:>8} => {} ({locale})", src, translated); + } + } + println!(); + + println!("Runtime numeric translation:"); + for i in 0..10usize { + for locale in &locales { + println!("{:>8} => {} ({locale})", i, tr!(i, locale = locale)); + } + } + println!(); +} From fdfc1ffa091c5931496634ff010c2b328410fbde Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 17:35:24 +0800 Subject: [PATCH 08/39] Update README.md for `tr!` --- README.md | 65 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b8517bb..3eb1329 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ > 🎯 Let's make I18n things to easy! -Rust I18n is a crate for loading localized text from a set of (YAML, JSON or TOML) mapping files. The mappings are converted into data readable by Rust programs at compile time, and then localized text can be loaded by simply calling the provided `t!` macro. +Rust I18n is a crate for loading localized text from a set of (YAML, JSON or TOML) mapping files. The mappings are converted into data readable by Rust programs at compile time, and then localized text can be loaded by simply calling the provided `t!` or `tr` macro. Unlike other I18n libraries, Rust I18n's goal is to provide a simple and easy-to-use API. @@ -13,7 +13,7 @@ The API of this crate is inspired by [ruby-i18n](https://github.com/ruby-i18n/i1 ## Features - Codegen on compile time for includes translations into binary. -- Global `t!` macro for loading localized text in everywhere. +- Global `t!` or `tr!` macro for loading localized text in everywhere. - Use YAML (default), JSON or TOML format for mapping localized text, and support mutiple files merging. - `cargo i18n` Command line tool for checking and extract untranslated texts into YAML files. - Support all localized texts in one file, or split into difference files by locale. @@ -30,7 +30,7 @@ rust-i18n = "3" Load macro and init translations in `lib.rs` or `main.rs`: ```rust,compile_fail,no_run -// Load I18n macro, for allow you use `t!` macro in anywhere. +// Load I18n macro, for allow you use `t!` or `tr!` macro in anywhere. #[macro_use] extern crate rust_i18n; @@ -54,14 +54,18 @@ i18n!("locales"); Or you can import by use directly: ```rust,no_run -// You must import in each files when you wants use `t!` macro. -use rust_i18n::t; +// You must import in each files when you wants use `t!` or `tr!` macro. +use rust_i18n::{t, tr}; rust_i18n::i18n!("locales"); fn main() { + // Find the translation for the string literal `Hello` using the manually provided key `hello`. println!("{}", t!("hello")); + // Or, find the translation for the string literal `Hello` using the auto-generated key. + println!("{}", tr!("Hello")); + // Use `available_locales!` method to get all available locales. println!("{:?}", rust_i18n::available_locales!()); } @@ -90,8 +94,11 @@ You can also split the each language into difference files, and you can choise ( ```yml _version: 1 -hello: 'Hello world' -messages.hello: 'Hello, %{name}' +hello: "Hello world" +messages.hello: "Hello, %{name}" + +# Key auto-generated by tr! +tr_4Cct6Q289b12SkvF47dXIx: "Hello, %{name}" ``` Or use JSON or TOML format, just rename the file to `en.json` or `en.toml`, and the content is like this: @@ -100,13 +107,19 @@ Or use JSON or TOML format, just rename the file to `en.json` or `en.toml`, and { "_version": 1, "hello": "Hello world", - "messages.hello": "Hello, %{name}" + "messages.hello": "Hello, %{name}", + + // Key auto-generated by tr! + "tr_4Cct6Q289b12SkvF47dXIx": "Hello, %{name}" } ``` ```toml hello = "Hello world" +# Key auto-generated by tr! +tr_4Cct6Q289b12SkvF47dXIx = "Hello, %{name}" + [messages] hello = "Hello, %{name}" ``` @@ -144,6 +157,11 @@ hello: messages.hello: en: Hello, %{name} zh-CN: 你好,%{name} + +# Key auto-generated by tr! +tr_4Cct6Q289b12SkvF47dXIx: + en: Hello, %{name} + zh-CN: 你好,%{name} ``` This is useful when you use [GitHub Copilot](https://github.com/features/copilot), after you write a first translated text, then Copilot will auto generate other locale's translations for you. @@ -152,18 +170,20 @@ This is useful when you use [GitHub Copilot](https://github.com/features/copilot ### Get Localized Strings in Rust -Import the `t!` macro from this crate into your current scope: +Import the `t!` or `tr` macro from this crate into your current scope: ```rust,no_run -use rust_i18n::t; +use rust_i18n::{t, tr}; ``` Then, simply use it wherever a localized string is needed: ```rust,no_run +# use rust_i18n::CowStr; # fn _rust_i18n_translate(locale: &str, key: &str) -> String { todo!() } +# fn _rust_i18n_try_translate<'r>(locale: &str, key: &'r str) -> Option> { todo!() } # fn main() { -use rust_i18n::t; +use rust_i18n::{t, tr}; t!("hello"); // => "Hello world" @@ -181,12 +201,29 @@ t!("messages.hello", locale = "zh-CN", name = "Jason", count = 2); t!("messages.hello", locale = "zh-CN", "name" => "Jason", "count" => 3 + 2); // => "你好,Jason (5)" + +tr!("Hello world"); +// => "Hello world" (Key `tr_3RnEdpgZvZ2WscJuSlQJkJ` for "Hello world") + +tr!("Hello world", locale = "de"); +// => "Hallo Welt!" (Key `tr_3RnEdpgZvZ2WscJuSlQJkJ` for "Hello world") + +tr!("Hello, %{name}", name = "world"); +// => "Hello, world" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hello, %{name}") + +tr!("Hello, %{name} and %{other}", name = "Foo", other ="Bar"); +// => "Hello, Foo and Bar" (Key `tr_3eULVGYoyiBuaM27F93Mo7` for "Hello, %{name} and %{other}") + +tr!("Hallo, %{name}", locale = "de", name = "Jason"); +// => "Hallo, Jason" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hallo, %{name}") # } ``` +💡 NOTE: The key `tr_3RnEdpgZvZ2WscJuSlQJkJ` is auto-generated by `tr!()`, and can be extract from source code to localized files with `rust-i18n-cli`. + ### Current Locale -You can use `rust_i18n::set_locale` to set the global locale at runtime, so that you don't have to specify the locale on each `t!` invocation. +You can use `rust_i18n::set_locale` to set the global locale at runtime, so that you don't have to specify the locale on each `t!` or `tr` invocation. ```rust rust_i18n::set_locale("zh-CN"); @@ -259,7 +296,7 @@ rust_i18n::i18n!("locales", backend = RemoteI18n::new()); This also will load local translates from ./locales path, but your own `RemoteI18n` will priority than it. -Now you call `t!` will lookup translates from your own backend first, if not found, will lookup from local files. +Now you call `t!` or `tr!` will lookup translates from your own backend first, if not found, will lookup from local files. ## Example @@ -269,7 +306,7 @@ A minimal example of using rust-i18n can be found [here](https://github.com/long I18n Ally is a VS Code extension for helping you translate your Rust project. -You can add [i18n-ally-custom-framework.yml](https://github.com/longbridgeapp/rust-i18n/blob/main/.vscode/i18n-ally-custom-framework.yml) to your project `.vscode` directory, and then use I18n Ally can parse `t!` marco to show translate text in VS Code editor. +You can add [i18n-ally-custom-framework.yml](https://github.com/longbridgeapp/rust-i18n/blob/main/.vscode/i18n-ally-custom-framework.yml) to your project `.vscode` directory, and then use I18n Ally can parse `t!` or `tr!` marco to show translate text in VS Code editor. ## Extractor From 24468c3c9327b3815b395eff5b74f9b52cff6e8d Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 19:36:46 +0800 Subject: [PATCH 09/39] Update to new rust-i18n::locale() with breaking changes Signed-off-by: Varphone Wong --- crates/macro/src/tr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs index e22c68d..008472c 100644 --- a/crates/macro/src/tr.rs +++ b/crates/macro/src/tr.rs @@ -179,7 +179,7 @@ impl Tr { let msg_val = self.msg.val; let msg_kind = self.msg.kind; let locale = self.locale.map_or_else( - || quote! { rust_i18n::locale().as_str() }, + || quote! { &rust_i18n::locale() }, |locale| quote! { #locale }, ); let keys: Vec<_> = self.args.keys().iter().map(|v| quote! { #v }).collect(); From 2c0af63174afa2009819228d628c06ae96bea091 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 19:56:11 +0800 Subject: [PATCH 10/39] Add concurrent testing for tr! --- tests/multi_threading.rs | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/multi_threading.rs b/tests/multi_threading.rs index 7d55b52..d10e959 100644 --- a/tests/multi_threading.rs +++ b/tests/multi_threading.rs @@ -2,7 +2,7 @@ use std::ops::Add; use std::thread::spawn; use std::time::{Duration, Instant}; -use rust_i18n::{set_locale, t}; +use rust_i18n::{set_locale, t, tr}; rust_i18n::i18n!("locales", fallback = "en"); @@ -32,3 +32,43 @@ fn test_load_and_store() { store.join().unwrap(); load.join().unwrap(); } + +#[test] +fn test_tr_concurrent() { + let end = Instant::now().add(Duration::from_secs(3)); + let store = spawn(move || { + let mut i = 0u32; + while Instant::now() < end { + for _ in 0..100 { + i = i.wrapping_add(1); + if i % 2 == 0 { + set_locale(&format!("en-{i}")); + } else { + set_locale(&format!("fr-{i}")); + } + } + } + }); + let tasks: Vec<_> = (0..4) + .map(|_| { + spawn(move || { + let locales = rust_i18n::available_locales!(); + let num_locales = locales.len(); + while Instant::now() < end { + for i in 0..100usize { + let m = i.checked_rem(num_locales).unwrap_or_default(); + if m == 0 { + tr!("hello"); + } else { + tr!("hello", locale = locales[m]); + } + } + } + }) + }) + .collect(); + store.join().unwrap(); + for task in tasks { + task.join().unwrap(); + } +} From f5a8322885832f21c769b7b2f906b4d3949c8341 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 22:27:58 +0800 Subject: [PATCH 11/39] Add tr_with_args (many-dynamic) and format! (many) benchmarks --- benches/bench.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/benches/bench.rs b/benches/bench.rs index c85f80b..ac10f03 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -144,6 +144,43 @@ fn bench_t(c: &mut Criterion) { ) }) }); + + c.bench_function("tr_with_args (many-dynamic)", |b| { + let msg = + r#"Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. + You live in %{city} %{zip}. + Your website is %{website}."# + .to_string(); + b.iter(|| { + tr!( + &msg, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ) + }) + }); + + c.bench_function("format! (many)", |b| { + b.iter(|| { + format!( + r#"Hello {name} %{surname}, your account id is {id}, email address is {email}. + You live in {city} {zip}. + Your website is {website}."#, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ); + }) + }); } criterion_group!(benches, bench_t); From ee5cd8c9657ff8c169d036871a1678d7e080bca9 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 22:28:18 +0800 Subject: [PATCH 12/39] examples/app-tr: Cleanup --- examples/app-tr/Cargo.toml | 1 - examples/app-tr/src/main.rs | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/app-tr/Cargo.toml b/examples/app-tr/Cargo.toml index f991521..0cb30a1 100644 --- a/examples/app-tr/Cargo.toml +++ b/examples/app-tr/Cargo.toml @@ -9,7 +9,6 @@ version = "0.1.0" env_logger = { version = "0.11", optional = true } log = { version = "0.4", optional = true } rust-i18n = { path = "../.." } -# rust-i18n-macro = { path = "../../crates/macro" } [features] log_tr_dyn = ["env_logger", "log", "rust-i18n/log_tr_dyn"] diff --git a/examples/app-tr/src/main.rs b/examples/app-tr/src/main.rs index 4671206..ad8e40d 100644 --- a/examples/app-tr/src/main.rs +++ b/examples/app-tr/src/main.rs @@ -23,8 +23,7 @@ fn main() { println!("String literals with patterns translation:"); for locale in &locales { println!( - "{} => {} ({})", - "Hello, %{name}!", + "Hello, %{{name}}! => {} ({})", tr!("Hello, %{name}!", name = "World", locale = locale), locale ); @@ -61,7 +60,7 @@ fn main() { println!(); println!("Runtime string translation:"); - let src_list = vec!["Apple", "Banana", "Orange"]; + let src_list = ["Apple", "Banana", "Orange"]; for src in src_list.iter() { for locale in &locales { let translated = tr!(*src, locale = locale); From 310dfc3bde1b9ab3de8926ec3aac7d39139fb0a8 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sat, 20 Jan 2024 23:27:31 +0800 Subject: [PATCH 13/39] Update SEO keywords to improve visibility of our package on crates.io --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 55df881..3ef3859 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ categories = ["localization", "internationalization"] description = "Rust I18n is use Rust codegen for load YAML file storage translations on compile time, and give you a t! macro for simply get translation texts." edition = "2021" exclude = ["crates", "tests"] -keywords = ["i18n", "yml", "localization", "internationalization"] +keywords = ["gettext", "i18n", "l10n", "intl", "internationalization", "localization", "tr", "translation", "yml"] license = "MIT" name = "rust-i18n" readme = "README.md" From 22740f1ccc6bbea17f2c260df476e57b5d466005 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 21 Jan 2024 01:51:14 +0800 Subject: [PATCH 14/39] Add colonsAdd colon assignment syntax for tr!, similar to struct field assignment For example: tr!("Hello, %{name}", name: "world"); --- crates/macro/src/lib.rs | 4 +++- crates/macro/src/tr.rs | 11 +++++++++-- tests/integration_tests.rs | 4 ++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/macro/src/lib.rs b/crates/macro/src/lib.rs index 90fdbc0..b895cf5 100644 --- a/crates/macro/src/lib.rs +++ b/crates/macro/src/lib.rs @@ -358,7 +358,9 @@ pub fn vakey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// // => "Hello, Foo and Bar" (Key `tr_3eULVGYoyiBuaM27F93Mo7` for "Hello, %{name} and %{other}") /// /// // With locale and variables -/// tr!("Hallo, %{name}", locale = "de", name = "Jason"); +/// tr!("Hallo, %{name}", locale = "de", name => "Jason"); // Arrow style +/// tr!("Hallo, %{name}", locale = "de", name = "Jason"); // Asignment style +/// tr!("Hallo, %{name}", locale = "de", name : "Jason"); // Colon style /// // => "Hallo, Jason" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hallo, %{name}") /// # } /// ``` diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs index 008472c..79e3158 100644 --- a/crates/macro/src/tr.rs +++ b/crates/macro/src/tr.rs @@ -27,8 +27,15 @@ impl syn::parse::Parse for Argument { .map(|v| v.to_string()) .or_else(|_| input.parse::().map(|v| v.value())) .map_err(|_| input.error("Expected a `string` literal or an identifier"))?; - let _ = input.parse::()?; - let _ = input.parse::]>>()?; + if input.peek(Token![=>]) { + let _ = input.parse::]>()?; + } else if input.peek(Token![=]) { + let _ = input.parse::()?; + } else if input.peek(Token![:]) { + let _ = input.parse::()?; + } else { + return Err(input.error("Expected `=>`, `=` or `:`")); + } let value = input.parse::()?; Ok(Self { name, value }) } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 879ed06..43082cd 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -225,7 +225,11 @@ mod tests { ); rust_i18n::set_locale("zh-CN"); + assert_eq!(tr!("Hello, %{name}!", name => "world"), "你好,world!"); assert_eq!(tr!("Hello, %{name}!", name = "world"), "你好,world!"); + assert_eq!(tr!("Hello, %{name}!", name : "world"), "你好,world!"); + assert_eq!(tr!("Hello, %{name}!", name: "world"), "你好,world!"); + assert_eq!(tr!("Hello, %{name}!", name:"world"), "你好,world!"); rust_i18n::set_locale("en"); assert_eq!(tr!("Hello, %{name}!", name = "world"), "Hello, world!"); From 834ec0a5d4ef8e14f62f0c9d987c950911a421b6 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 21 Jan 2024 02:43:45 +0800 Subject: [PATCH 15/39] Add new option --tr for rust-i18n-cli to support manual add translation for tr! Usage: cargo i18n --tr "Hello, world" --- crates/cli/src/main.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 8f94fdf..852613c 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,8 +1,10 @@ use anyhow::Error; use clap::{Args, Parser}; +use rust_i18n_support::TrKey; use std::{collections::HashMap, path::Path}; +use rust_i18n_extract::extractor::Message; use rust_i18n_extract::{extractor, generator, iter}; mod config; @@ -27,6 +29,22 @@ struct I18nArgs { /// Extract all untranslated I18n texts from source code #[arg(default_value = "./")] source: Option, + /// Add a translation to the localize file for tr! + #[arg(long, default_value = None)] + tr: Option>, +} + +/// Add translations to the localize file for tr! +fn add_translations(list: &[String], results: &mut HashMap) { + for item in list { + let index = results.len(); + results.entry(item.tr_key()).or_insert(Message { + key: item.clone(), + index, + is_tr: true, + locations: vec![], + }); + } } fn main() -> Result<(), Error> { @@ -42,6 +60,10 @@ fn main() -> Result<(), Error> { extractor::extract(&mut results, path, source) })?; + if let Some(list) = args.tr { + add_translations(&list, &mut results); + } + let mut messages: Vec<_> = results.iter().collect(); messages.sort_by_key(|(_k, m)| m.index); From 85b33e7a3170764e2a37942086b3e35788380d92 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 21 Jan 2024 11:20:49 +0800 Subject: [PATCH 16/39] Rename feature `log_tr_dyn` to `log-tr-dyn` --- Cargo.toml | 2 +- crates/macro/Cargo.toml | 2 +- crates/macro/src/tr.rs | 4 ++-- examples/app-tr/Cargo.toml | 2 +- examples/app-tr/src/main.rs | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3ef3859..88fb76f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,4 +48,4 @@ harness = false name = "bench" [features] -log_tr_dyn = ["rust-i18n-macro/log_tr_dyn"] +log-tr-dyn = ["rust-i18n-macro/log-tr-dyn"] diff --git a/crates/macro/Cargo.toml b/crates/macro/Cargo.toml index 1c8ed5a..596b87a 100644 --- a/crates/macro/Cargo.toml +++ b/crates/macro/Cargo.toml @@ -27,4 +27,4 @@ rust-i18n = { path = "../.." } proc-macro = true [features] -log_tr_dyn = [] +log-tr-dyn = [] diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs index 79e3158..77cb77b 100644 --- a/crates/macro/src/tr.rs +++ b/crates/macro/src/tr.rs @@ -221,7 +221,7 @@ impl Tr { } } Messagekind::ExprCall | Messagekind::ExprClosure | Messagekind::ExprMacro => { - let logging = if cfg!(feature = "log_tr_dyn") { + let logging = if cfg!(feature = "log-tr-dyn") { quote! { log::debug!("tr: missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); } @@ -264,7 +264,7 @@ impl Tr { | Messagekind::ExprReference | Messagekind::ExprUnary | Messagekind::Ident => { - let logging = if cfg!(feature = "log_tr_dyn") { + let logging = if cfg!(feature = "log-tr-dyn") { quote! { log::debug!("tr: missing: {} => {:?} @ {}:{}", msg_key, #msg_val, file!(), line!()); } diff --git a/examples/app-tr/Cargo.toml b/examples/app-tr/Cargo.toml index 0cb30a1..b812f17 100644 --- a/examples/app-tr/Cargo.toml +++ b/examples/app-tr/Cargo.toml @@ -11,7 +11,7 @@ log = { version = "0.4", optional = true } rust-i18n = { path = "../.." } [features] -log_tr_dyn = ["env_logger", "log", "rust-i18n/log_tr_dyn"] +log-tr-dyn = ["env_logger", "log", "rust-i18n/log-tr-dyn"] [package.metadata.i18n] available-locales = ["en", "zh-CN"] diff --git a/examples/app-tr/src/main.rs b/examples/app-tr/src/main.rs index ad8e40d..c69e2f1 100644 --- a/examples/app-tr/src/main.rs +++ b/examples/app-tr/src/main.rs @@ -2,7 +2,7 @@ use rust_i18n::tr; rust_i18n::i18n!("locales"); -#[cfg(feature = "log_tr_dyn")] +#[cfg(feature = "log-tr-dyn")] fn set_logger() { env_logger::builder() .filter_level(log::LevelFilter::Debug) @@ -10,7 +10,7 @@ fn set_logger() { .init(); } -#[cfg(not(feature = "log_tr_dyn"))] +#[cfg(not(feature = "log-tr-dyn"))] fn set_logger() {} fn main() { From 18276116f86c27132859037f6c0ebf27c41fe2ca Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 21 Jan 2024 11:24:34 +0800 Subject: [PATCH 17/39] Change tr! missing log level to warn --- crates/macro/src/tr.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs index 77cb77b..bda692c 100644 --- a/crates/macro/src/tr.rs +++ b/crates/macro/src/tr.rs @@ -223,7 +223,7 @@ impl Tr { Messagekind::ExprCall | Messagekind::ExprClosure | Messagekind::ExprMacro => { let logging = if cfg!(feature = "log-tr-dyn") { quote! { - log::debug!("tr: missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); + log::warn!("tr: missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); } } else { quote! {} @@ -266,7 +266,7 @@ impl Tr { | Messagekind::Ident => { let logging = if cfg!(feature = "log-tr-dyn") { quote! { - log::debug!("tr: missing: {} => {:?} @ {}:{}", msg_key, #msg_val, file!(), line!()); + log::warn!("tr: missing: {} => {:?} @ {}:{}", msg_key, #msg_val, file!(), line!()); } } else { quote! {} From 658c6803a3af9d270fd4a1fb91a0310e24abbe93 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 21 Jan 2024 11:47:14 +0800 Subject: [PATCH 18/39] Refactor Tr --- crates/macro/src/tr.rs | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs index bda692c..eef737f 100644 --- a/crates/macro/src/tr.rs +++ b/crates/macro/src/tr.rs @@ -196,6 +196,13 @@ impl Tr { .iter() .map(|v| quote! { format!("{}", #v) }) .collect(); + let logging = if cfg!(feature = "log-tr-dyn") { + quote! { + log::warn!("tr: missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); + } + } else { + quote! {} + }; match msg_kind { Messagekind::Literal => { if self.args.is_empty() { @@ -221,13 +228,6 @@ impl Tr { } } Messagekind::ExprCall | Messagekind::ExprClosure | Messagekind::ExprMacro => { - let logging = if cfg!(feature = "log-tr-dyn") { - quote! { - log::warn!("tr: missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); - } - } else { - quote! {} - }; if self.args.is_empty() { quote! { { @@ -264,21 +264,16 @@ impl Tr { | Messagekind::ExprReference | Messagekind::ExprUnary | Messagekind::Ident => { - let logging = if cfg!(feature = "log-tr-dyn") { - quote! { - log::warn!("tr: missing: {} => {:?} @ {}:{}", msg_key, #msg_val, file!(), line!()); - } - } else { - quote! {} - }; if self.args.is_empty() { quote! { { let msg_key = rust_i18n::TrKey::tr_key(&#msg_key); + let msg_val = #msg_val; if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { translated } else { - rust_i18n::CowStr::from(#msg_val).into_inner() + #logging + rust_i18n::CowStr::from(msg_val).into_inner() } } } @@ -286,6 +281,7 @@ impl Tr { quote! { { let msg_key = rust_i18n::TrKey::tr_key(&#msg_key); + let msg_val = #msg_val; let keys = &[#(#keys),*]; let values = &[#(#values),*]; if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { @@ -293,7 +289,7 @@ impl Tr { std::borrow::Cow::from(replaced) } else { #logging - let replaced = rust_i18n::replace_patterns(&rust_i18n::CowStr::from(#msg_val).into_inner(), keys, values); + let replaced = rust_i18n::replace_patterns(&rust_i18n::CowStr::from(msg_val).into_inner(), keys, values); std::borrow::Cow::from(replaced) } } From 20ba42139c40dd2136e498b4280dace30e15ca84 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 21 Jan 2024 11:50:18 +0800 Subject: [PATCH 19/39] examples/app-tr: Add log-tr-dyn demos --- examples/app-tr/src/main.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/app-tr/src/main.rs b/examples/app-tr/src/main.rs index c69e2f1..bcc9fc0 100644 --- a/examples/app-tr/src/main.rs +++ b/examples/app-tr/src/main.rs @@ -76,4 +76,13 @@ fn main() { } } println!(); + + if cfg!(feature = "log-tr-dyn") { + println!("Runtime string missing with logging:"); + for locale in &locales { + let msg = "Foo Bar".to_string(); + println!("{:>8} => {} ({locale})", &msg, tr!(&msg, locale = locale)); + } + println!(); + } } From 0449d7b4de63425ba26fab86cb4991015a776d3b Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 21 Jan 2024 13:26:59 +0800 Subject: [PATCH 20/39] Improve the help documentation for the `--tr` option in `rust-i18n-cli` --- crates/cli/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 852613c..149a3a6 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -29,8 +29,8 @@ struct I18nArgs { /// Extract all untranslated I18n texts from source code #[arg(default_value = "./")] source: Option, - /// Add a translation to the localize file for tr! - #[arg(long, default_value = None)] + /// Add a translation to the localize file for `tr!` + #[arg(long, default_value = None, name = "TEXT")] tr: Option>, } From 8f431d03264a501b0222e2cbbcd192d8239544a2 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 21 Jan 2024 22:15:18 +0800 Subject: [PATCH 21/39] Add specifiers support to tr! and remove colons assignment style Added: tr!("Hello, %{name} and %{other}", name = "Foo", other = 123 : {:08}); Removed: tr!("Hallo, %{name}", locale = "de", name : "Jason"); --- README.md | 7 ++- crates/macro/src/lib.rs | 5 +- crates/macro/src/tr.rs | 94 ++++++++++++++++++++++++++++++++----- examples/app-tr/src/main.rs | 9 ++++ tests/integration_tests.rs | 42 +++++++++++++++-- 5 files changed, 139 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3eb1329..89a9101 100644 --- a/README.md +++ b/README.md @@ -211,11 +211,14 @@ tr!("Hello world", locale = "de"); tr!("Hello, %{name}", name = "world"); // => "Hello, world" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hello, %{name}") -tr!("Hello, %{name} and %{other}", name = "Foo", other ="Bar"); +tr!("Hello, %{name} and %{other}", name = "Foo", other = "Bar"); // => "Hello, Foo and Bar" (Key `tr_3eULVGYoyiBuaM27F93Mo7` for "Hello, %{name} and %{other}") -tr!("Hallo, %{name}", locale = "de", name = "Jason"); +tr!("Hello, %{name}", locale = "de", name = "Jason"); // => "Hallo, Jason" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hallo, %{name}") + +tr!("Hello, %{name}, you serial number is: %{sn}", name = "Jason", sn = 123 : {:08}); +// => "Hello, Jason, you serial number is: 000000123" # } ``` diff --git a/crates/macro/src/lib.rs b/crates/macro/src/lib.rs index b895cf5..7cfb713 100644 --- a/crates/macro/src/lib.rs +++ b/crates/macro/src/lib.rs @@ -357,10 +357,13 @@ pub fn vakey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// tr!("Hello, %{name} and %{other}", name = "Foo", other ="Bar"); /// // => "Hello, Foo and Bar" (Key `tr_3eULVGYoyiBuaM27F93Mo7` for "Hello, %{name} and %{other}") /// +/// // With variables and specifiers +/// tr!("Hello, %{name} and %{other}", name = "Foo", other = 123 : {:08}); +/// // => "Hello, Foo and 00000123" ( +/// /// // With locale and variables /// tr!("Hallo, %{name}", locale = "de", name => "Jason"); // Arrow style /// tr!("Hallo, %{name}", locale = "de", name = "Jason"); // Asignment style -/// tr!("Hallo, %{name}", locale = "de", name : "Jason"); // Colon style /// // => "Hallo, Jason" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hallo, %{name}") /// # } /// ``` diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs index eef737f..bc47966 100644 --- a/crates/macro/src/tr.rs +++ b/crates/macro/src/tr.rs @@ -1,17 +1,61 @@ use quote::{quote, ToTokens}; use rust_i18n_support::TrKey; -use syn::{parse::discouraged::Speculative, Expr, ExprMacro, Ident, LitStr, Token}; +use syn::{parse::discouraged::Speculative, token::Brace, Expr, ExprMacro, Ident, LitStr, Token}; + +#[derive(Clone)] +pub enum Value { + Expr(Expr), + Ident(Ident), +} + +impl From for Value { + fn from(expr: Expr) -> Self { + Self::Expr(expr) + } +} + +impl From for Value { + fn from(ident: Ident) -> Self { + Self::Ident(ident) + } +} + +impl quote::ToTokens for Value { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + match self { + Self::Expr(expr) => expr.to_tokens(tokens), + Self::Ident(ident) => ident.to_tokens(tokens), + } + } +} + +impl syn::parse::Parse for Value { + fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + if let Ok(expr) = fork.parse::() { + input.advance_to(&fork); + return Ok(expr.into()); + } + let fork = input.fork(); + if let Ok(expr) = fork.parse::() { + input.advance_to(&fork); + return Ok(expr.into()); + } + Err(input.error("Expected a expression or an identifier")) + } +} pub struct Argument { pub name: String, - pub value: Expr, + pub value: Value, + pub specifiers: Option, } impl Argument { #[allow(dead_code)] pub fn value_string(&self) -> String { match &self.value { - Expr::Lit(expr_lit) => match &expr_lit.lit { + Value::Expr(Expr::Lit(expr_lit)) => match &expr_lit.lit { syn::Lit::Str(lit_str) => lit_str.value(), _ => self.value.to_token_stream().to_string(), }, @@ -31,13 +75,31 @@ impl syn::parse::Parse for Argument { let _ = input.parse::]>()?; } else if input.peek(Token![=]) { let _ = input.parse::()?; - } else if input.peek(Token![:]) { - let _ = input.parse::()?; } else { - return Err(input.error("Expected `=>`, `=` or `:`")); + return Err(input.error("Expected `=>` or `=`")); } - let value = input.parse::()?; - Ok(Self { name, value }) + let value = input.parse()?; + let specifiers = if input.peek(Token![:]) { + let _ = input.parse::()?; + if input.peek(Brace) { + let content; + let _ = syn::braced!(content in input); + let mut specifiers = String::new(); + while let Ok(s) = content.parse::() { + specifiers.push_str(&s.to_string()); + } + Some(specifiers) + } else { + None + } + } else { + None + }; + Ok(Self { + name, + value, + specifiers, + }) } } @@ -55,7 +117,8 @@ impl Arguments { self.args.iter().map(|arg| arg.name.clone()).collect() } - pub fn values(&self) -> Vec { + #[allow(dead_code)] + pub fn values(&self) -> Vec { self.args.iter().map(|arg| arg.value.clone()).collect() } } @@ -177,7 +240,7 @@ impl syn::parse::Parse for Messsage { pub(crate) struct Tr { pub msg: Messsage, pub args: Arguments, - pub locale: Option, + pub locale: Option, } impl Tr { @@ -192,9 +255,16 @@ impl Tr { let keys: Vec<_> = self.args.keys().iter().map(|v| quote! { #v }).collect(); let values: Vec<_> = self .args - .values() + .as_ref() .iter() - .map(|v| quote! { format!("{}", #v) }) + .map(|v| { + let value = &v.value; + let sepecifiers = v + .specifiers + .as_ref() + .map_or("{}".to_owned(), |s| format!("{{{}}}", s)); + quote! { format!(#sepecifiers, #value) } + }) .collect(); let logging = if cfg!(feature = "log-tr-dyn") { quote! { diff --git a/examples/app-tr/src/main.rs b/examples/app-tr/src/main.rs index bcc9fc0..58e2622 100644 --- a/examples/app-tr/src/main.rs +++ b/examples/app-tr/src/main.rs @@ -85,4 +85,13 @@ fn main() { } println!(); } + + println!("String literals with specified args translation:"); + for i in (0..10000).step_by(50) { + println!( + "Zero padded number: %{{count}} => {}", + tr!("Zero padded number: %{count}", count = i : {:08}), + ); + } + println!(); } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 43082cd..4fe636d 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -227,9 +227,6 @@ mod tests { rust_i18n::set_locale("zh-CN"); assert_eq!(tr!("Hello, %{name}!", name => "world"), "你好,world!"); assert_eq!(tr!("Hello, %{name}!", name = "world"), "你好,world!"); - assert_eq!(tr!("Hello, %{name}!", name : "world"), "你好,world!"); - assert_eq!(tr!("Hello, %{name}!", name: "world"), "你好,world!"); - assert_eq!(tr!("Hello, %{name}!", name:"world"), "你好,world!"); rust_i18n::set_locale("en"); assert_eq!(tr!("Hello, %{name}!", name = "world"), "Hello, world!"); @@ -317,6 +314,45 @@ mod tests { ); } + #[test] + fn test_tr_with_specified_args() { + #[derive(Debug)] + struct Foo { + #[allow(unused)] + bar: usize, + } + assert_eq!( + tr!("Any: %{value}", value = Foo { bar : 1 } : {:?}), + "Any: Foo { bar: 1 }" + ); + let foo = Foo { bar: 2 }; + assert_eq!( + tr!("Any: %{value}", value = foo : {:?}), + "Any: Foo { bar: 2 }" + ); + assert_eq!( + tr!("Any: %{value}", value = &foo : {:?}), + "Any: Foo { bar: 2 }" + ); + assert_eq!(tr!("Any: %{value}", value = foo.bar : {:?}), "Any: 2"); + assert_eq!( + tr!("You have %{count} messages.", count => 123 : {:08}), + "You have 00000123 messages." + ); + assert_eq!( + tr!("You have %{count} messages.", count => 100 + 23 : {:>8}), + "You have 123 messages." + ); + assert_eq!( + tr!("You have %{count} messages.", count => 1 * 100 + 23 : {:08}, locale = "zh-CN"), + "你收到了 00000123 条新消息。" + ); + assert_eq!( + tr!("You have %{count} messages.", count => 100 + 23 * 1 / 1 : {:>8}, locale = "zh-CN"), + "你收到了 123 条新消息。" + ); + } + #[test] fn test_with_merge_file() { rust_i18n::set_locale("en"); From be107865a05fbca2f1ce55120f98fc01ad6bd400 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 14:01:56 +0800 Subject: [PATCH 22/39] Add CowStr::as_str() --- crates/support/src/cow_str.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/support/src/cow_str.rs b/crates/support/src/cow_str.rs index 2544807..7b7fc44 100644 --- a/crates/support/src/cow_str.rs +++ b/crates/support/src/cow_str.rs @@ -7,6 +7,10 @@ use std::sync::Arc; pub struct CowStr<'a>(Cow<'a, str>); impl<'a> CowStr<'a> { + pub fn as_str(&self) -> &str { + self.0.as_ref() + } + pub fn into_inner(self) -> Cow<'a, str> { self.0 } @@ -57,6 +61,13 @@ impl<'a> From<&'a str> for CowStr<'a> { } } +impl<'a> From<&&'a str> for CowStr<'a> { + #[inline] + fn from(s: &&'a str) -> Self { + Self(Cow::Borrowed(s)) + } +} + impl<'a> From> for CowStr<'a> { #[inline] fn from(s: Arc<&'a str>) -> Self { From ff3ebd90b2827c2a5cbce3863abaa3e0bbb96468 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 14:03:06 +0800 Subject: [PATCH 23/39] Rename TrKey to MinifyKey --- crates/support/Cargo.toml | 2 +- crates/support/src/lib.rs | 7 +- crates/support/src/minify_key.rs | 166 +++++++++++++++++++++++++++++++ crates/support/src/tr_key.rs | 106 -------------------- 4 files changed, 172 insertions(+), 109 deletions(-) create mode 100644 crates/support/src/minify_key.rs delete mode 100644 crates/support/src/tr_key.rs diff --git a/crates/support/Cargo.toml b/crates/support/Cargo.toml index eab6f28..d59ac21 100644 --- a/crates/support/Cargo.toml +++ b/crates/support/Cargo.toml @@ -16,7 +16,7 @@ proc-macro2 = "1.0" serde = "1" serde_json = "1" serde_yaml = "0.8" -siphasher = "1.0.0" +siphasher = "1.0" toml = "0.7.4" normpath = "1.1.1" lazy_static = "1" diff --git a/crates/support/src/lib.rs b/crates/support/src/lib.rs index 7663a5a..cd2cc3f 100644 --- a/crates/support/src/lib.rs +++ b/crates/support/src/lib.rs @@ -6,11 +6,14 @@ use std::{collections::HashMap, path::Path}; mod atomic_str; mod backend; mod cow_str; -mod tr_key; +mod minify_key; pub use atomic_str::AtomicStr; pub use backend::{Backend, BackendExt, SimpleBackend}; pub use cow_str::CowStr; -pub use tr_key::{TrKey, TrKeyNumeric, TR_KEY_PREFIX}; +pub use minify_key::{ + minify_key, MinifyKey, DEFAULT_MINIFY_KEY, DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, + DEFAULT_MINIFY_KEY_THRESH, +}; type Locale = String; type Value = serde_json::Value; diff --git a/crates/support/src/minify_key.rs b/crates/support/src/minify_key.rs new file mode 100644 index 0000000..02475e6 --- /dev/null +++ b/crates/support/src/minify_key.rs @@ -0,0 +1,166 @@ +use once_cell::sync::Lazy; +use siphasher::sip128::SipHasher13; +use std::borrow::Cow; + +/// The default value of `minify_key` feature. +pub const DEFAULT_MINIFY_KEY: bool = false; + +/// The length of auto-generated translation key +pub const DEFAULT_MINIFY_KEY_LEN: usize = 24; + +/// The prefix of auto-generated translation key +pub const DEFAULT_MINIFY_KEY_PREFIX: &str = ""; + +/// The minimum length of the value to be generated the translation key +pub const DEFAULT_MINIFY_KEY_THRESH: usize = 127; + +// The hasher for generate the literal translation key +static TR_KEY_HASHER: Lazy = Lazy::new(SipHasher13::new); + +/// Calculate a 128-bit siphash of a value. +pub fn hash128 + ?Sized>(value: &T) -> u128 { + TR_KEY_HASHER.hash(value.as_ref()).as_u128() +} + +/// Generate a translation key from a value. +/// +/// # Arguments +/// +/// * `value` - The value to be generated. +/// * `key_len` - The length of the translation key. +/// * `prefix` - The prefix of the translation key. +/// * `threshold` - The minimum length of the value to be generated. +/// +/// # Returns +/// +/// * If `value.len() <= threshold` then returns the origin value. +/// * Otherwise, returns a base62 encoded 128 bits hashed translation key. +/// +pub fn minify_key<'r>( + value: &'r str, + key_len: usize, + prefix: &str, + threshold: usize, +) -> Cow<'r, str> { + if value.len() <= threshold { + return Cow::Borrowed(value); + } + let encoded = base62::encode(hash128(value)); + let key_len = key_len.min(encoded.len()); + format!("{}{}", prefix, &encoded[..key_len]).into() +} + +/// A trait for generating translation key from a value. +pub trait MinifyKey<'a> { + /// Generate translation key from a value. + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str>; +} + +impl<'a> MinifyKey<'a> for str { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for &str { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for String { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + if self.len() <= threshold { + return Cow::Borrowed(self); + } + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for &String { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + if self.len() <= threshold { + return Cow::from(*self); + } + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for Cow<'a, str> { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + if self.len() <= threshold { + return Cow::Borrowed(self); + } + minify_key(self, key_len, prefix, threshold) + } +} + +impl<'a> MinifyKey<'a> for &Cow<'a, str> { + #[inline] + fn minify_key(&'a self, key_len: usize, prefix: &str, threshold: usize) -> Cow<'a, str> { + if self.len() <= threshold { + return Cow::Borrowed(*self); + } + minify_key(self, key_len, prefix, threshold) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_minify_key() { + let msg = "Hello, world!"; + assert_eq!( + minify_key(msg, 24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + let msg = format!("Hello, world!"); + assert_eq!( + minify_key(&msg, 24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 128), + "Hello, world!" + ); + let msg = &msg; + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + let msg = Cow::Owned("Hello, world!".to_owned()); + assert_eq!( + minify_key(&msg, 24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "1LokVzuiIrh1xByyZG4wjZ" + ); + assert_eq!( + msg.minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 128), + "Hello, world!" + ); + assert_eq!("".minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), ""); + assert_eq!( + "1".minify_key(24, DEFAULT_MINIFY_KEY_PREFIX, 0), + "knx7vOJBRfzgQvNfEkbEi" + ); + assert_eq!("1".minify_key(24, "t_", 0), "t_knx7vOJBRfzgQvNfEkbEi"); + } +} diff --git a/crates/support/src/tr_key.rs b/crates/support/src/tr_key.rs deleted file mode 100644 index 6278677..0000000 --- a/crates/support/src/tr_key.rs +++ /dev/null @@ -1,106 +0,0 @@ -use once_cell::sync::Lazy; -use siphasher::sip128::SipHasher13; - -/// The prefix of auto-generated literal translation key -pub const TR_KEY_PREFIX: &str = "tr_"; - -// The hasher for generate the literal translation key -static TR_KEY_HASHER: Lazy = Lazy::new(SipHasher13::new); - -pub trait TrKeyNumeric: std::fmt::Display { - fn tr_key_numeric(&self) -> String { - format!("{}N_{}", TR_KEY_PREFIX, self) - } -} - -/// A trait for generating translation key from a value. -pub trait TrKey { - fn tr_key(&self) -> String; -} - -macro_rules! impl_tr_key_for_numeric { - ($typ:ty) => { - impl TrKeyNumeric for $typ {} - impl TrKey for $typ { - #[inline] - fn tr_key(&self) -> String { - self.tr_key_numeric() - } - } - }; -} - -macro_rules! impl_tr_key_for_signed_numeric { - ($typ:ty) => { - impl TrKeyNumeric for $typ {} - impl TrKey for $typ { - #[inline] - fn tr_key(&self) -> String { - (*self as u128).tr_key_numeric() - } - } - }; -} - -impl_tr_key_for_numeric!(u8); -impl_tr_key_for_numeric!(u16); -impl_tr_key_for_numeric!(u32); -impl_tr_key_for_numeric!(u64); -impl_tr_key_for_numeric!(u128); -impl_tr_key_for_numeric!(usize); -impl_tr_key_for_signed_numeric!(i8); -impl_tr_key_for_signed_numeric!(i16); -impl_tr_key_for_signed_numeric!(i32); -impl_tr_key_for_signed_numeric!(i64); -impl_tr_key_for_signed_numeric!(i128); -impl_tr_key_for_signed_numeric!(isize); - -impl TrKey for [u8] { - #[inline] - fn tr_key(&self) -> String { - let hash = TR_KEY_HASHER.hash(self).as_u128(); - format!("{}{}", TR_KEY_PREFIX, base62::encode(hash)) - } -} - -impl TrKey for str { - #[inline] - fn tr_key(&self) -> String { - self.as_bytes().tr_key() - } -} - -impl TrKey for &str { - #[inline] - fn tr_key(&self) -> String { - self.as_bytes().tr_key() - } -} - -impl TrKey for String { - #[inline] - fn tr_key(&self) -> String { - self.as_bytes().tr_key() - } -} - -impl TrKey for &String { - #[inline] - fn tr_key(&self) -> String { - self.as_bytes().tr_key() - } -} - -impl<'a> TrKey for std::borrow::Cow<'a, str> { - #[inline] - fn tr_key(&self) -> String { - self.as_bytes().tr_key() - } -} - -impl<'a> TrKey for &std::borrow::Cow<'a, str> { - #[inline] - fn tr_key(&self) -> String { - self.as_bytes().tr_key() - } -} From 3a8111bdc87cb04e5b8756ba1b8ab9046460fe76 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 14:06:10 +0800 Subject: [PATCH 24/39] Add `minify_key` supports to `in18n!` and `tr!` --- crates/macro/Cargo.toml | 4 +- crates/macro/src/lib.rs | 176 ++++++++++++--- crates/macro/src/mikey.rs | 49 +++++ crates/macro/src/tr.rs | 442 +++++++++++++++++++++++--------------- 4 files changed, 462 insertions(+), 209 deletions(-) create mode 100644 crates/macro/src/mikey.rs diff --git a/crates/macro/Cargo.toml b/crates/macro/Cargo.toml index 596b87a..fe9dfad 100644 --- a/crates/macro/Cargo.toml +++ b/crates/macro/Cargo.toml @@ -18,7 +18,7 @@ rust-i18n-support = { path = "../support", version = "3.0.0" } serde = "1" serde_json = "1" serde_yaml = "0.8" -syn = { version = "2.0.18", features = ["full"] } +syn = { version = "2.0.18", features = ["full", "extra-traits"] } [dev-dependencies] rust-i18n = { path = "../.." } @@ -27,4 +27,4 @@ rust-i18n = { path = "../.." } proc-macro = true [features] -log-tr-dyn = [] +log-missing = [] diff --git a/crates/macro/src/lib.rs b/crates/macro/src/lib.rs index 7cfb713..aba5842 100644 --- a/crates/macro/src/lib.rs +++ b/crates/macro/src/lib.rs @@ -1,14 +1,22 @@ use quote::quote; -use rust_i18n_support::{is_debug, load_locales}; +use rust_i18n_support::{ + is_debug, load_locales, DEFAULT_MINIFY_KEY, DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, + DEFAULT_MINIFY_KEY_THRESH, +}; use std::collections::HashMap; -use syn::{parse_macro_input, Expr, Ident, LitStr, Token}; +use syn::{parse_macro_input, Expr, Ident, LitBool, LitStr, Token}; +mod mikey; mod tr; struct Args { locales_path: String, fallback: Option>, extend: Option, + minify_key: bool, + minify_key_len: usize, + minify_key_prefix: String, + minify_key_thresh: usize, } impl Args { @@ -46,6 +54,36 @@ impl Args { Ok(()) } + fn consume_minify_key(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { + let lit_bool = input.parse::()?; + self.minify_key = lit_bool.value; + Ok(()) + } + + fn consume_minify_key_len(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { + let lit_int = input.parse::()?; + self.minify_key_len = lit_int.base10_parse()?; + Ok(()) + } + + fn consume_minify_key_prefix( + &mut self, + input: syn::parse::ParseStream, + ) -> syn::parse::Result<()> { + let lit_str = input.parse::()?; + self.minify_key_prefix = lit_str.value(); + Ok(()) + } + + fn consume_minify_key_thresh( + &mut self, + input: syn::parse::ParseStream, + ) -> syn::parse::Result<()> { + let lit_int = input.parse::()?; + self.minify_key_thresh = lit_int.base10_parse()?; + Ok(()) + } + fn consume_options(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { let ident = input.parse::()?.to_string(); input.parse::()?; @@ -58,6 +96,18 @@ impl Args { let val = input.parse::()?; self.extend = Some(val); } + "minify_key" => { + self.consume_minify_key(input)?; + } + "minify_key_len" => { + self.consume_minify_key_len(input)?; + } + "minify_key_prefix" => { + self.consume_minify_key_prefix(input)?; + } + "minify_key_thresh" => { + self.consume_minify_key_thresh(input)?; + } _ => {} } @@ -97,6 +147,10 @@ impl syn::parse::Parse for Args { locales_path: String::from("locales"), fallback: None, extend: None, + minify_key: DEFAULT_MINIFY_KEY, + minify_key_len: DEFAULT_MINIFY_KEY_LEN, + minify_key_prefix: DEFAULT_MINIFY_KEY_PREFIX.to_owned(), + minify_key_thresh: DEFAULT_MINIFY_KEY_THRESH, }; if lookahead.peek(LitStr) { @@ -117,7 +171,16 @@ impl syn::parse::Parse for Args { /// /// This will load all translations by glob `**/*.yml` from the given path, default: `${CARGO_MANIFEST_DIR}/locales`. /// -/// Attribute `fallback` for set the fallback locale, if present `t` macro will use it as the fallback locale. +/// # Attributes +/// +/// - `fallback` for set the fallback locale, if present [`t!`](macro.t.html) macro will use it as the fallback locale. +/// - `backend` for set the backend, if present [`t!`](macro.t.html) macro will use it as the backend. +/// - `minify_key` for enable/disable minify key, default: [`DEFAULT_MINIFY_KEY`](constant.DEFAULT_MINIFY_KEY.html). +/// - `minify_key_len` for set the minify key length, default: [`DEFAULT_MINIFY_KEY_LEN`](constant.DEFAULT_MINIFY_KEY_LEN.html), +/// * The range of available values is from `0` to `24`. +/// - `minify_key_prefix` for set the minify key prefix, default: [`DEFAULT_MINIFY_KEY_PREFIX`](constant.DEFAULT_MINIFY_KEY_PREFIX.html). +/// - `minify_key_thresh` for set the minify key threshold, default: [`DEFAULT_MINIFY_KEY_THRESH`](constant.DEFAULT_MINIFY_KEY_THRESH.html). +/// * If the length of the value is less than or equal to this value, the value will not be minified. /// /// ```no_run /// # use rust_i18n::i18n; @@ -133,6 +196,13 @@ impl syn::parse::Parse for Args { /// # fn v4() { /// i18n!("locales", fallback = ["en", "es"]); /// # } +/// # fn v5() { +/// i18n!("locales", fallback = ["en", "es"], +/// minify_key = true, +/// minify_key_len = 12, +/// minify_key_prefix = "T.", +/// minify_key_thresh = 64); +/// # } /// ``` #[proc_macro] pub fn i18n(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -197,9 +267,13 @@ fn generate_code( quote! {} }; - // result + let minify_key = args.minify_key; + let minify_key_len = args.minify_key_len; + let minify_key_prefix = args.minify_key_prefix; + let minify_key_thresh = args.minify_key_thresh; + quote! { - use rust_i18n::{BackendExt, CowStr, TrKey}; + use rust_i18n::{BackendExt, CowStr, MinifyKey}; use std::borrow::Cow; /// I18n backend instance @@ -215,6 +289,10 @@ fn generate_code( }); static _RUST_I18N_FALLBACK_LOCALE: Option<&[&'static str]> = #fallback; + static _RUST_I18N_MINIFY_KEY: bool = #minify_key; + static _RUST_I18N_MINIFY_KEY_LEN: usize = #minify_key_len; + static _RUST_I18N_MINIFY_KEY_PREFIX: &str = #minify_key_prefix; + static _RUST_I18N_MINIFY_KEY_THRESH: usize = #minify_key_thresh; /// Lookup fallback locales /// @@ -267,6 +345,27 @@ fn generate_code( locales.sort(); locales } + + #[allow(unused_macros)] + macro_rules! __rust_i18n_t { + ($($all_tokens:tt)*) => { + rust_i18n::tr!($($all_tokens)*, _minify_key = #minify_key, _minify_key_len = #minify_key_len, _minify_key_prefix = #minify_key_prefix, _minify_key_thresh = #minify_key_thresh) + } + } + + #[allow(unused_macros)] + macro_rules! __rust_i18n_tkv { + ($msg:literal) => { + { + let val = $msg; + let key = rust_i18n::mikey!($msg, #minify_key_len, #minify_key_prefix, #minify_key_thresh); + (key, val) + } + } + } + + pub(crate) use __rust_i18n_t as _rust_i18n_t; + pub(crate) use __rust_i18n_tkv as _rust_i18n_tkv; } } @@ -308,33 +407,42 @@ pub fn vakey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { } } -/// Get I18n text with literals supports. +/// A procedural macro that generates a translation key from a value. /// -/// This macro first checks if a translation exists for the input string. -/// If it does, it returns the translated string. -/// If it does not, it returns the input string literal. -/// -/// # Variants +/// # Arguments /// -/// This macro has several variants that allow for different use cases: +/// * `value` - The value to be generated. +/// * `key_len` - The length of the translation key. +/// * `prefix` - The prefix of the translation key. +/// * `threshold` - The minimum length of the value to be generated. /// -/// * `tr!("foo")`: -/// Translates the string "foo" using the current locale. +/// # Returns /// -/// * `tr!("foo", locale = "en")`: -/// Translates the string "foo" using the specified locale "en". +/// * If `value.len() <= threshold` then returns the origin value. +/// * Otherwise, returns a base62 encoded 128 bits hashed translation key. /// -/// * `tr!("foo", locale = "en", a = 1, b = "Foo")`: -/// Translates the string "foo" using the specified locale "en" and replaces the patterns "{a}" and "{b}" in the string with "1" and "Foo" respectively. +/// # Example /// -/// * `tr!("foo %{a} %{b}", a = "bar", b = "baz")`: -/// Translates the string "foo %{a} %{b}" using the current locale and replaces the patterns "{a}" and "{b}" in the string with "bar" and "baz" respectively. +/// ```no_run +/// # use rust_i18n::mikey; +/// # fn v1() { +/// mikey!("Hello world", 12, "T.", 64); +/// // => "Hello world" /// -/// * `tr!("foo %{a} %{b}", locale = "en", "a" => "bar", "b" => "baz")`: -/// Translates the string "foo %{a} %{b}" using the specified locale "en" and replaces the patterns "{a}" and "{b}" in the string with "bar" and "baz" respectively. +/// mikey!("Hello world", 12, "T.", 5); +/// // => "T.1b9d6bcd" +/// # } +/// ``` +#[proc_macro] +pub fn mikey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + parse_macro_input!(input as mikey::MiKey).into() +} + +/// A procedural macro that retrieves the i18n text for the `t!` macro. /// -/// * `tr!("foo %{a} %{b}", "a" => "bar", "b" => "baz")`: -/// Translates the string "foo %{a} %{b}" using the current locale and replaces the patterns "{a}" and "{b}" in the string with "bar" and "baz" respectively. +/// This macro first checks if a translation exists for the input string. +/// If it does, it returns the translated string. +/// If it does not, it returns the input value. /// /// # Examples /// @@ -345,26 +453,26 @@ pub fn vakey(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// # fn main() { /// // Simple get text with current locale /// tr!("Hello world"); -/// // => "Hello world" (Key `tr_3RnEdpgZvZ2WscJuSlQJkJ` for "Hello world") +/// // => "Hello world" /// /// // Get a special locale's text /// tr!("Hello world", locale = "de"); -/// // => "Hallo Welt!" (Key `tr_3RnEdpgZvZ2WscJuSlQJkJ` for "Hello world") +/// // => "Hallo Welt!" /// /// // With variables -/// tr!("Hello, %{name}", name = "world"); -/// // => "Hello, world" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hello, %{name}") -/// tr!("Hello, %{name} and %{other}", name = "Foo", other ="Bar"); -/// // => "Hello, Foo and Bar" (Key `tr_3eULVGYoyiBuaM27F93Mo7` for "Hello, %{name} and %{other}") +/// tr!("Hello, %{name}", name = "world"); // Asignment style +/// tr!("Hello, %{name}", name => "world"); // Arrow style +/// // => "Hello, world" +/// tr!("Hello, %{name} and %{other}", name = "Foo", other = "Bar"); +/// // => "Hello, Foo and Bar" /// /// // With variables and specifiers /// tr!("Hello, %{name} and %{other}", name = "Foo", other = 123 : {:08}); -/// // => "Hello, Foo and 00000123" ( +/// // => "Hello, Foo and 00000123" /// /// // With locale and variables -/// tr!("Hallo, %{name}", locale = "de", name => "Jason"); // Arrow style -/// tr!("Hallo, %{name}", locale = "de", name = "Jason"); // Asignment style -/// // => "Hallo, Jason" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hallo, %{name}") +/// tr!("Hallo, %{name}", locale = "de", name => "Jason"); +/// // => "Hallo, Jason" /// # } /// ``` #[proc_macro] diff --git a/crates/macro/src/mikey.rs b/crates/macro/src/mikey.rs new file mode 100644 index 0000000..e71939c --- /dev/null +++ b/crates/macro/src/mikey.rs @@ -0,0 +1,49 @@ +use quote::quote; +use rust_i18n_support::minify_key; +use syn::Token; + +/// A type representing the `mikey!` proc macro. +#[derive(Clone, Debug, Default)] +pub struct MiKey { + msg: String, + len: usize, + prefix: String, + threshold: usize, +} + +impl MiKey { + fn into_token_stream(self) -> proc_macro2::TokenStream { + let key = minify_key(&self.msg, self.len, &self.prefix, self.threshold); + quote! { #key } + } +} + +impl syn::parse::Parse for MiKey { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + let msg = input.parse::()?.value(); + let _comma = input.parse::()?; + let len: usize = input.parse::()?.base10_parse()?; + let _comma = input.parse::()?; + let prefix = input.parse::()?.value(); + let _comma = input.parse::()?; + let threshold: usize = input.parse::()?.base10_parse()?; + Ok(Self { + msg, + len, + prefix, + threshold, + }) + } +} + +impl From for proc_macro::TokenStream { + fn from(val: MiKey) -> Self { + val.into_token_stream().into() + } +} + +impl From for proc_macro2::TokenStream { + fn from(val: MiKey) -> Self { + val.into_token_stream() + } +} diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs index bc47966..1ba9bef 100644 --- a/crates/macro/src/tr.rs +++ b/crates/macro/src/tr.rs @@ -1,13 +1,60 @@ use quote::{quote, ToTokens}; -use rust_i18n_support::TrKey; -use syn::{parse::discouraged::Speculative, token::Brace, Expr, ExprMacro, Ident, LitStr, Token}; +use rust_i18n_support::{ + MinifyKey, DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, DEFAULT_MINIFY_KEY_THRESH, +}; +use syn::{parse::discouraged::Speculative, token::Brace, Expr, Ident, LitStr, Token}; -#[derive(Clone)] +#[derive(Clone, Debug, Default)] pub enum Value { + #[default] + Empty, Expr(Expr), Ident(Ident), } +impl Value { + fn is_expr_lit_str(&self) -> bool { + if let Self::Expr(Expr::Lit(expr_lit)) = self { + if let syn::Lit::Str(_) = &expr_lit.lit { + return true; + } + } + false + } + + fn is_expr_tuple(&self) -> bool { + if let Self::Expr(Expr::Tuple(_)) = self { + return true; + } + false + } + + fn to_string(&self) -> Option { + if let Self::Expr(Expr::Lit(expr_lit)) = self { + if let syn::Lit::Str(lit_str) = &expr_lit.lit { + return Some(lit_str.value()); + } + } + None + } + + fn to_tupled_token_streams( + &self, + ) -> syn::parse::Result<(proc_macro2::TokenStream, proc_macro2::TokenStream)> { + if let Self::Expr(Expr::Tuple(expr_tuple)) = self { + if expr_tuple.elems.len() == 2 { + let first = expr_tuple.elems.first().map(|v| quote! { #v }).unwrap(); + let last = expr_tuple.elems.last().map(|v| quote! { #v }).unwrap(); + return Ok((first, last)); + } + } + Err(syn::Error::new_spanned( + self, + "Expected a tuple with two elements", + )) + } +} + impl From for Value { fn from(expr: Expr) -> Self { Self::Expr(expr) @@ -23,8 +70,12 @@ impl From for Value { impl quote::ToTokens for Value { fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { match self { - Self::Expr(expr) => expr.to_tokens(tokens), - Self::Ident(ident) => ident.to_tokens(tokens), + Self::Empty => {} + Self::Expr(expr) => match expr { + Expr::Path(path) => quote! { &#path }.to_tokens(tokens), + expr => expr.to_tokens(tokens), + }, + Self::Ident(ident) => quote! { &#ident }.to_tokens(tokens), } } } @@ -45,6 +96,7 @@ impl syn::parse::Parse for Value { } } +#[derive(Clone, Default)] pub struct Argument { pub name: String, pub value: Value, @@ -62,15 +114,33 @@ impl Argument { _ => self.value.to_token_stream().to_string(), } } + + fn try_ident(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let ident = fork.parse::()?; + input.advance_to(&fork); + Ok(ident.to_string()) + } + + fn try_literal(input: syn::parse::ParseStream) -> syn::parse::Result { + let fork = input.fork(); + let lit = fork.parse::()?; + input.advance_to(&fork); + Ok(lit.value()) + } } impl syn::parse::Parse for Argument { fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { - let name = input - .parse::() - .map(|v| v.to_string()) - .or_else(|_| input.parse::().map(|v| v.value())) + // Ignore leading commas. + while input.peek(Token![,]) { + let _ = input.parse::()?; + } + // Parse the argument name. + let name = Self::try_ident(input) + .or_else(|_| Self::try_literal(input)) .map_err(|_| input.error("Expected a `string` literal or an identifier"))?; + // Parse the separator between the name and the value. if input.peek(Token![=>]) { let _ = input.parse::]>()?; } else if input.peek(Token![=]) { @@ -78,7 +148,9 @@ impl syn::parse::Parse for Argument { } else { return Err(input.error("Expected `=>` or `=`")); } + // Parse the argument value. let value = input.parse()?; + // Parse the specifiers [optinal]. let specifiers = if input.peek(Token![:]) { let _ = input.parse::()?; if input.peek(Brace) { @@ -113,6 +185,10 @@ impl Arguments { self.args.is_empty() } + pub fn iter(&self) -> impl Iterator { + self.args.iter() + } + pub fn keys(&self) -> Vec { self.args.iter().map(|arg| arg.name.clone()).collect() } @@ -145,109 +221,183 @@ impl syn::parse::Parse for Arguments { } } -#[derive(Default)] -pub enum Messagekind { - #[default] - Literal, - Expr, - ExprCall, - ExprClosure, - ExprMacro, - ExprReference, - ExprUnary, - Ident, -} - #[derive(Default)] pub struct Messsage { pub key: proc_macro2::TokenStream, - pub val: proc_macro2::TokenStream, - pub kind: Messagekind, + pub val: Value, } impl Messsage { fn try_exp(input: syn::parse::ParseStream) -> syn::parse::Result { let fork = input.fork(); let expr = fork.parse::()?; - let key = quote! { #expr }; - let val = quote! { #expr }; input.advance_to(&fork); - let kind = match expr { - Expr::Call(_) => Messagekind::ExprCall, - Expr::Closure(_) => Messagekind::ExprClosure, - Expr::Macro(_) => Messagekind::ExprMacro, - Expr::Reference(_) => Messagekind::ExprReference, - Expr::Unary(_) => Messagekind::ExprUnary, - _ => Messagekind::Expr, - }; - Ok(Self { key, val, kind }) - } - #[allow(dead_code)] - fn try_exp_macro(input: syn::parse::ParseStream) -> syn::parse::Result { - let fork = input.fork(); - let expr = fork.parse::()?; - let key = quote! { #expr }; - let val = quote! { #expr }; - input.advance_to(&fork); Ok(Self { - key, - val, - kind: Messagekind::ExprMacro, + key: Default::default(), + val: Value::Expr(expr), }) } fn try_ident(input: syn::parse::ParseStream) -> syn::parse::Result { let fork = input.fork(); let ident = fork.parse::()?; - let key = quote! { #ident }; - let val = quote! { #ident }; - input.advance_to(&fork); - Ok(Self { - key, - val, - kind: Messagekind::Ident, - }) - } - - fn try_litreal(input: syn::parse::ParseStream) -> syn::parse::Result { - let fork = input.fork(); - let lit_str = fork.parse::()?; - let key = lit_str.value().tr_key(); - let key = quote! { #key }; - let val = quote! { #lit_str }; input.advance_to(&fork); Ok(Self { - key, - val, - kind: Messagekind::Literal, + key: Default::default(), + val: Value::Ident(ident), }) } } impl syn::parse::Parse for Messsage { fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { - let result = Self::try_litreal(input) - // .or_else(|_| Self::try_exp_macro(input)) - .or_else(|_| Self::try_exp(input)) - .or_else(|_| Self::try_ident(input))?; + let result = Self::try_exp(input).or_else(|_| Self::try_ident(input))?; Ok(result) } } -/// A type representing the `tr!` macro. -#[derive(Default)] +/// A type representing the `tr!` proc macro. pub(crate) struct Tr { pub msg: Messsage, pub args: Arguments, pub locale: Option, + pub minify_key: bool, + pub minify_key_len: usize, + pub minify_key_prefix: String, + pub minify_key_thresh: usize, } impl Tr { + fn new() -> Self { + Self { + msg: Messsage::default(), + args: Arguments::default(), + locale: None, + minify_key: false, + minify_key_len: DEFAULT_MINIFY_KEY_LEN, + minify_key_prefix: DEFAULT_MINIFY_KEY_PREFIX.into(), + minify_key_thresh: DEFAULT_MINIFY_KEY_THRESH, + } + } + + fn parse_minify_key(value: &Value) -> syn::parse::Result { + if let Value::Expr(Expr::Lit(expr_lit)) = value { + match &expr_lit.lit { + syn::Lit::Bool(lit_bool) => { + return Ok(lit_bool.value); + } + syn::Lit::Str(lit_str) => { + let value = lit_str.value(); + if ["true", "false", "yes", "no"].contains(&value.as_str()) { + return Ok(["true", "yes"].contains(&value.as_str())); + } + } + _ => {} + } + } + Err(syn::Error::new_spanned( + value, + "`_minify_key` Expected a string literal in `true`, `false`, `yes`, `no`", + )) + } + + fn parse_minify_key_len(value: &Value) -> syn::parse::Result { + if let Value::Expr(Expr::Lit(expr_lit)) = value { + if let syn::Lit::Int(lit_int) = &expr_lit.lit { + return Ok(lit_int.base10_parse().unwrap()); + } + } + Err(syn::Error::new_spanned( + value, + "`_minify_key_len` Expected a integer literal", + )) + } + + fn parse_minify_key_prefix(value: &Value) -> syn::parse::Result { + if let Value::Expr(Expr::Lit(expr_lit)) = value { + if let syn::Lit::Str(lit_str) = &expr_lit.lit { + return Ok(lit_str.value()); + } + } + Err(syn::Error::new_spanned( + value, + "`_minify_key_prefix` Expected a string literal", + )) + } + + fn parse_minify_key_thresh(value: &Value) -> syn::parse::Result { + if let Value::Expr(Expr::Lit(expr_lit)) = value { + if let syn::Lit::Int(lit_int) = &expr_lit.lit { + return Ok(lit_int.base10_parse().unwrap()); + } + } + Err(syn::Error::new_spanned( + value, + "`_minify_key_threshold` Expected a integer literal", + )) + } + + fn filter_arguments(&mut self) -> syn::parse::Result<()> { + for arg in self.args.iter() { + match arg.name.as_str() { + "locale" => { + self.locale = Some(arg.value.clone()); + } + "_minify_key" => { + self.minify_key = Self::parse_minify_key(&arg.value)?; + } + "_minify_key_len" => { + self.minify_key_len = Self::parse_minify_key_len(&arg.value)?; + } + "_minify_key_prefix" => { + self.minify_key_prefix = Self::parse_minify_key_prefix(&arg.value)?; + } + "_minify_key_thresh" => { + self.minify_key_thresh = Self::parse_minify_key_thresh(&arg.value)?; + } + _ => {} + } + } + + self.args.as_mut().retain(|v| { + ![ + "locale", + "_minify_key", + "_minify_key_len", + "_minify_key_prefix", + "_minify_key_thresh", + ] + .contains(&v.name.as_str()) + }); + + Ok(()) + } + fn into_token_stream(self) -> proc_macro2::TokenStream { - let msg_key = self.msg.key; - let msg_val = self.msg.val; - let msg_kind = self.msg.kind; + let (msg_key, msg_val) = if self.minify_key && self.msg.val.is_expr_lit_str() { + let msg_val = self.msg.val.to_string().unwrap(); + let msg_key = MinifyKey::minify_key( + &msg_val, + self.minify_key_len, + self.minify_key_prefix.as_str(), + self.minify_key_thresh, + ); + (quote! { #msg_key }, quote! { #msg_val }) + } else if self.minify_key && self.msg.val.is_expr_tuple() { + self.msg.val.to_tupled_token_streams().unwrap() + } else if self.minify_key { + let minify_key_len = self.minify_key_len; + let minify_key_prefix = self.minify_key_prefix; + let minify_key_thresh = self.minify_key_thresh; + let msg_val = self.msg.val.to_token_stream(); + let msg_key = quote! { rust_i18n::MinifyKey::minify_key(&msg_val, #minify_key_len, #minify_key_prefix, #minify_key_thresh) }; + (msg_key, msg_val) + } else { + let msg_val = self.msg.val.to_token_stream(); + let msg_key = quote! { &msg_val }; + (msg_key, msg_val) + }; let locale = self.locale.map_or_else( || quote! { &rust_i18n::locale() }, |locale| quote! { #locale }, @@ -266,128 +416,74 @@ impl Tr { quote! { format!(#sepecifiers, #value) } }) .collect(); - let logging = if cfg!(feature = "log-tr-dyn") { + let logging = if cfg!(feature = "log-missing") { quote! { - log::warn!("tr: missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); + log::log!(target: "rust-i18n", log::Level::Warn, "missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); } } else { quote! {} }; - match msg_kind { - Messagekind::Literal => { - if self.args.is_empty() { - quote! { - crate::_rust_i18n_try_translate(#locale, #msg_key).unwrap_or_else(|| std::borrow::Cow::from(#msg_val)) - } - } else { - quote! { - { - let msg_key = #msg_key; - let msg_val = #msg_val; - let keys = &[#(#keys),*]; - let values = &[#(#values),*]; - if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { - let replaced = rust_i18n::replace_patterns(&translated, keys, values); - std::borrow::Cow::from(replaced) - } else { - let replaced = rust_i18n::replace_patterns(&rust_i18n::CowStr::from(msg_val).into_inner(), keys, values); - std::borrow::Cow::from(replaced) - } - } + if self.args.is_empty() { + quote! { + { + let msg_val = #msg_val; + let msg_key = #msg_key; + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { + translated.into() + } else { + #logging + rust_i18n::CowStr::from(msg_val).into_inner() } } } - Messagekind::ExprCall | Messagekind::ExprClosure | Messagekind::ExprMacro => { - if self.args.is_empty() { - quote! { - { - let msg_val = #msg_val; - let msg_key = rust_i18n::TrKey::tr_key(&msg_val); - if let Some(translated) = crate::_rust_i18n_try_translate(#locale, msg_key) { - translated - } else { - #logging - std::borrow::Cow::from(msg_val) - } - } - } - } else { - quote! { - { - let msg_val = #msg_val; - let msg_key = rust_i18n::TrKey::tr_key(&msg_val); - let keys = &[#(#keys),*]; - let values = &[#(#values),*]; - if let Some(translated) = crate::_rust_i18n_try_translate(#locale, msg_key) { - let replaced = rust_i18n::replace_patterns(&translated, keys, values); - std::borrow::Cow::from(replaced) - } else { - #logging - let replaced = rust_i18n::replace_patterns(&rust_i18n::CowStr::from(msg_val).into_inner(), keys, values); - std::borrow::Cow::from(replaced) - } - } + } else { + quote! { + { + let msg_val = #msg_val; + let msg_key = #msg_key; + let keys = &[#(#keys),*]; + let values = &[#(#values),*]; + { + if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { + let replaced = rust_i18n::replace_patterns(&translated, keys, values); + std::borrow::Cow::from(replaced) + } else { + #logging + let replaced = rust_i18n::replace_patterns(rust_i18n::CowStr::from(msg_val).as_str(), keys, values); + std::borrow::Cow::from(replaced) } } - } - Messagekind::Expr - | Messagekind::ExprReference - | Messagekind::ExprUnary - | Messagekind::Ident => { - if self.args.is_empty() { - quote! { - { - let msg_key = rust_i18n::TrKey::tr_key(&#msg_key); - let msg_val = #msg_val; - if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { - translated - } else { - #logging - rust_i18n::CowStr::from(msg_val).into_inner() - } - } - } - } else { - quote! { - { - let msg_key = rust_i18n::TrKey::tr_key(&#msg_key); - let msg_val = #msg_val; - let keys = &[#(#keys),*]; - let values = &[#(#values),*]; - if let Some(translated) = crate::_rust_i18n_try_translate(#locale, &msg_key) { - let replaced = rust_i18n::replace_patterns(&translated, keys, values); - std::borrow::Cow::from(replaced) - } else { - #logging - let replaced = rust_i18n::replace_patterns(&rust_i18n::CowStr::from(msg_val).into_inner(), keys, values); - std::borrow::Cow::from(replaced) - } - } - } } } } } } +impl Default for Tr { + fn default() -> Self { + Self::new() + } +} + impl syn::parse::Parse for Tr { fn parse(input: syn::parse::ParseStream) -> syn::parse::Result { let msg = input.parse::()?; let comma = input.parse::>()?; - let (args, locale) = if comma.is_some() { - let mut args = input.parse::()?; - let locale = args - .as_ref() - .iter() - .find(|v| v.name == "locale") - .map(|v| v.value.clone()); - args.as_mut().retain(|v| v.name != "locale"); - (args, locale) + let args = if comma.is_some() { + input.parse::()? } else { - (Arguments::default(), None) + Arguments::default() + }; + + let mut result = Self { + msg, + args, + ..Self::new() }; - Ok(Self { msg, args, locale }) + result.filter_arguments()?; + + Ok(result) } } From 4d78704ed6f7f0512058e415df858dc1b734effc Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 14:08:05 +0800 Subject: [PATCH 25/39] Add `minify_key` supports to `t!` --- Cargo.toml | 2 +- src/lib.rs | 132 ++++++++++++++++++++++++++++++++--------------------- 2 files changed, 81 insertions(+), 53 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 88fb76f..bf3996a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,4 +48,4 @@ harness = false name = "bench" [features] -log-tr-dyn = ["rust-i18n-macro/log-tr-dyn"] +log-missing = ["rust-i18n-macro/log-missing"] diff --git a/src/lib.rs b/src/lib.rs index 94edf32..b0f67d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,8 +6,11 @@ use once_cell::sync::Lazy; #[doc(hidden)] pub use once_cell; -pub use rust_i18n_macro::{i18n, tr, vakey}; -pub use rust_i18n_support::{AtomicStr, Backend, BackendExt, CowStr, SimpleBackend, TrKey}; +pub use rust_i18n_macro::{i18n, mikey, tr, vakey}; +pub use rust_i18n_support::{ + AtomicStr, Backend, BackendExt, CowStr, MinifyKey, SimpleBackend, DEFAULT_MINIFY_KEY, + DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, DEFAULT_MINIFY_KEY_THRESH, +}; static CURRENT_LOCALE: Lazy = Lazy::new(|| AtomicStr::from("en")); @@ -89,71 +92,96 @@ pub fn replace_patterns(input: &str, patterns: &[&str], values: &[String]) -> St /// Get I18n text /// +/// This macro forwards to the `crate::_rust_i18n_t!` macro, which is generated by the [`i18n!`] macro. +/// +/// # Arguments +/// +/// * `expr` - The key or message for translation. +/// - A key usually looks like `"foo.bar.baz"`. +/// - A literal message usually looks like `"Hello, world!"`. +/// - The variable names in the message should be wrapped in `%{}`, like `"Hello, %{name}!"`. +/// - Dynamic messages are also supported, such as `t!(format!("Hello, {}!", name))`. +/// However, if `minify_key` is enabled, the entire message will be hashed and used as a key for every lookup, which may consume more CPU cycles. +/// * `locale` - The locale to use. If not specified, the current locale will be used. +/// * `args` - The arguments to be replaced in the translated text. +/// - These should be passed in the format `key = value` or `key => value`. +/// - Alternatively, you can specify the value format using the `key = value : {:format_specifier}` syntax. +/// For example, `key = value : {:08}` will format the value as a zero-padded string with a length of 8. +/// +/// # Example +/// /// ```no_run /// #[macro_use] extern crate rust_i18n; -/// # fn _rust_i18n_translate(locale: &str, key: &str) -> String { todo!() } +/// +/// # macro_rules! t { ($($all:tt)*) => {} } /// # fn main() { /// // Simple get text with current locale -/// t!("greeting"); // greeting: "Hello world" => "Hello world" +/// t!("greeting"); +/// // greeting: "Hello world" => "Hello world" +/// /// // Get a special locale's text -/// t!("greeting", locale = "de"); // greeting: "Hallo Welt!" => "Hallo Welt!" +/// t!("greeting", locale = "de"); +/// // greeting: "Hallo Welt!" => "Hallo Welt!" /// /// // With variables -/// t!("messages.hello", name = "world"); // messages.hello: "Hello, {name}" => "Hello, world" -/// t!("messages.foo", name = "Foo", other ="Bar"); // messages.foo: "Hello, {name} and {other}" => "Hello, Foo and Bar" +/// t!("messages.hello", name = "world"); +/// // messages.hello: "Hello, %{name}" => "Hello, world" +/// t!("messages.foo", name = "Foo", other ="Bar"); +/// // messages.foo: "Hello, %{name} and %{other}" => "Hello, Foo and Bar" +/// +/// // With variables and format specifiers +/// t!("Hello, %{name}, you serial number is: %{sn}", name = "Jason", sn = 123 : {:08}); +/// // => "Hello, Jason, you serial number is: 000000123" /// /// // With locale and variables -/// t!("messages.hello", locale = "de", name = "Jason"); // messages.hello: "Hallo, {name}" => "Hallo, Jason" +/// t!("messages.hello", locale = "de", name = "Jason"); +/// // messages.hello: "Hallo, %{name}" => "Hallo, Jason" /// # } /// ``` #[macro_export] #[allow(clippy::crate_in_macro_def)] macro_rules! t { - // t!("foo") - ($key:expr) => { - crate::_rust_i18n_translate(&rust_i18n::locale(), $key) - }; - - // t!("foo", locale = "en") - ($key:expr, locale = $locale:expr) => { - crate::_rust_i18n_translate($locale, $key) - }; - - // t!("foo", locale = "en", a = 1, b = "Foo") - ($key:expr, locale = $locale:expr, $($var_name:tt = $var_val:expr),+ $(,)?) => { - { - let message = crate::_rust_i18n_translate($locale, $key); - let patterns: &[&str] = &[ - $(rust_i18n::vakey!($var_name)),+ - ]; - let values = &[ - $(format!("{}", $var_val)),+ - ]; - - let output = rust_i18n::replace_patterns(message.as_ref(), patterns, values); - std::borrow::Cow::from(output) - } - }; - - // t!("foo %{a} %{b}", a = "bar", b = "baz") - ($key:expr, $($var_name:tt = $var_val:expr),+ $(,)?) => { - { - t!($key, locale = &rust_i18n::locale(), $($var_name = $var_val),*) - } - }; - - // t!("foo %{a} %{b}", locale = "en", "a" => "bar", "b" => "baz") - ($key:expr, locale = $locale:expr, $($var_name:tt => $var_val:expr),+ $(,)?) => { - { - t!($key, locale = $locale, $($var_name = $var_val),*) - } - }; + ($($all:tt)*) => { + crate::_rust_i18n_t!($($all)*) + } +} - // t!("foo %{a} %{b}", "a" => "bar", "b" => "baz") - ($key:expr, $($var_name:tt => $var_val:expr),+ $(,)?) => { - { - t!($key, locale = &rust_i18n::locale(), $($var_name = $var_val),*) - } +/// A macro that generates a translation key and corresponding value pair from a given input value. +/// +/// It's useful when you want to use a long string as a key, but you don't want to type it twice. +/// +/// # Arguments +/// +/// * `msg` - The input value. +/// +/// # Returns +/// +/// A tuple of `(key, msg)`. +/// +/// # Example +/// +/// ```no_run +/// use rust_i18n::{t, tkv}; +/// +/// # macro_rules! t { ($($all:tt)*) => { } } +/// # macro_rules! tkv { ($($all:tt)*) => { (1,2) } } +/// +/// let (key, msg) = tkv!( +/// r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque sed nisi leo. Donec commodo in ex at aliquam. Nunc in aliquam arcu. Fusce mollis metus orci, ut sagittis erat lobortis sed. Morbi quis arcu ultrices turpis finibus tincidunt non in purus. Donec gravida condimentum sapien. Duis iaculis fermentum congue. Quisque blandit libero a lacus auctor vestibulum. Nunc efficitur sollicitudin nisi, sit amet tristique lectus mollis non. Praesent sit amet erat volutpat, pharetra orci eget, rutrum felis. Sed elit augue, imperdiet eu facilisis vel, finibus vel urna. Duis quis neque metus. +/// +/// Mauris suscipit bibendum mattis. Vestibulum eu augue diam. Morbi dapibus tempus viverra. Sed aliquam turpis eget justo ornare maximus vitae et tortor. Donec semper neque sit amet sapien congue scelerisque. Maecenas bibendum imperdiet dolor interdum facilisis. Integer non diam tempus, pharetra ex at, euismod diam. Ut enim turpis, sagittis in iaculis ut, finibus et sem. Suspendisse a felis euismod neque euismod placerat. Praesent ipsum libero, porta vel egestas quis, aliquet vitae lorem. Nullam vel pharetra erat, sit amet sodales leo."# +/// ); +/// // Use the `key` and `msg` in `t!` macro +/// t!((key, msg), bar = "foo"); +/// // ... +/// // In other parts of the code, you can reuse the same `key` and `msg` +/// t!((key, msg), bar = "baz"); +/// ``` +#[macro_export] +#[allow(clippy::crate_in_macro_def)] +macro_rules! tkv { + ($msg:literal) => { + crate::_rust_i18n_tkv!($msg) }; } From 1be4d47cc15c9947748d24b2dae01a6871e1a6ec Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 14:09:20 +0800 Subject: [PATCH 26/39] Add `minify_key` supports to `rust-i18n-cli` --- crates/cli/src/config.rs | 23 +++++++++++ crates/cli/src/main.rs | 73 ++++++++++++++++++++++++++++----- crates/extract/Cargo.toml | 2 +- crates/extract/src/attrs.rs | 47 +++++++++++++++++++++ crates/extract/src/extractor.rs | 47 +++++++++++++++------ crates/extract/src/generator.rs | 3 +- crates/extract/src/lib.rs | 1 + 7 files changed, 170 insertions(+), 26 deletions(-) create mode 100644 crates/extract/src/attrs.rs diff --git a/crates/cli/src/config.rs b/crates/cli/src/config.rs index 883a433..888d9e6 100644 --- a/crates/cli/src/config.rs +++ b/crates/cli/src/config.rs @@ -4,6 +4,7 @@ //! See `Manifest::from_slice`. use itertools::Itertools; +use rust_i18n_extract::attrs::I18nAttrs; use serde::{Deserialize, Serialize}; use std::fs; use std::io; @@ -19,6 +20,8 @@ pub struct I18nConfig { pub available_locales: Vec, #[serde(default = "load_path")] pub load_path: String, + #[serde(flatten)] + pub attrs: I18nAttrs, } fn default_locale() -> String { @@ -45,6 +48,7 @@ impl Default for I18nConfig { default_locale: "en".to_string(), available_locales: vec!["en".to_string()], load_path: "./locales".to_string(), + attrs: I18nAttrs::default(), } } } @@ -87,12 +91,20 @@ fn test_parse() { default-locale = "en" available-locales = ["zh-CN"] load-path = "./my-locales" + minify-key = true + minify-key-len = 12 + minify-key-prefix = "T." + minify-key-thresh = 16 "#; let cfg = parse(contents).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); assert_eq!(cfg.load_path, "./my-locales"); + assert_eq!(cfg.attrs.minify_key, true); + assert_eq!(cfg.attrs.minify_key_len, 12); + assert_eq!(cfg.attrs.minify_key_prefix, "T."); + assert_eq!(cfg.attrs.minify_key_thresh, 16); let contents = r#" [i18n] @@ -103,12 +115,14 @@ fn test_parse() { assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN", "de"]); assert_eq!(cfg.load_path, "./my-locales"); + assert_eq!(cfg.attrs, I18nAttrs::default()); let contents = ""; let cfg = parse(contents).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en"]); assert_eq!(cfg.load_path, "./locales"); + assert_eq!(cfg.attrs, I18nAttrs::default()); } #[test] @@ -118,12 +132,20 @@ fn test_parse_with_metadata() { default-locale = "en" available-locales = ["zh-CN"] load-path = "./my-locales" + minify-key = true + minify-key-len = 12 + minify-key-prefix = "T." + minify-key-thresh = 16 "#; let cfg = parse(contents).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); assert_eq!(cfg.load_path, "./my-locales"); + assert_eq!(cfg.attrs.minify_key, true); + assert_eq!(cfg.attrs.minify_key_len, 12); + assert_eq!(cfg.attrs.minify_key_prefix, "T."); + assert_eq!(cfg.attrs.minify_key_thresh, 16); } #[test] @@ -134,6 +156,7 @@ fn test_load_default() { assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en"]); assert_eq!(cfg.load_path, "./locales"); + assert_eq!(cfg.attrs, I18nAttrs::default()); } #[test] diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 149a3a6..295f807 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,6 +1,7 @@ use anyhow::Error; use clap::{Args, Parser}; -use rust_i18n_support::TrKey; +use rust_i18n_extract::attrs::I18nAttrs; +use rust_i18n_support::MinifyKey; use std::{collections::HashMap, path::Path}; @@ -29,19 +30,69 @@ struct I18nArgs { /// Extract all untranslated I18n texts from source code #[arg(default_value = "./")] source: Option, - /// Add a translation to the localize file for `tr!` - #[arg(long, default_value = None, name = "TEXT")] - tr: Option>, + /// Manually add a translation to the localization file. + /// + /// This is useful for non-literal values in the `t!` macro. + /// + /// For example, if you have `t!(format!("Hello, {}!", "world"))` in your code, + /// you can add a translation for it using `-t "Hello, world!"`, + /// or provide a translated message using `-t "Hello, world! => Hola, world!"`. + /// + /// NOTE: The whitespace before and after the key and value will be trimmed. + #[arg(short, long, default_value = None, name = "TEXT", num_args(1..), value_parser = translate_value_parser, verbatim_doc_comment)] + translate: Option>, +} + +/// Remove quotes from a string at the start and end. +fn remove_quotes(s: &str) -> &str { + let mut start = 0; + let mut end = s.len(); + if s.starts_with('"') { + start += 1; + } + if s.ends_with('"') { + end -= 1; + } + &s[start..end] +} + +/// Parse a string of the form "key => value" into a tuple. +fn translate_value_parser(s: &str) -> Result<(String, String), std::io::Error> { + if let Some((key, msg)) = s.split_once("=>") { + let key = remove_quotes(key.trim()); + let msg = remove_quotes(msg.trim()); + Ok((key.to_owned(), msg.to_owned())) + } else { + Ok((s.to_owned(), s.to_owned())) + } } /// Add translations to the localize file for tr! -fn add_translations(list: &[String], results: &mut HashMap) { +fn add_translations( + list: &[(String, String)], + results: &mut HashMap, + attrs: &I18nAttrs, +) { + let I18nAttrs { + minify_key, + minify_key_len, + minify_key_prefix, + minify_key_thresh, + } = attrs; for item in list { let index = results.len(); - results.entry(item.tr_key()).or_insert(Message { - key: item.clone(), + let key = if *minify_key { + let hashed_key = + item.0 + .minify_key(*minify_key_len, minify_key_prefix, *minify_key_thresh); + hashed_key.to_string() + } else { + item.0.clone() + }; + results.entry(key).or_insert(Message { + key: item.1.clone(), index, - is_tr: true, + minify_key: *minify_key, locations: vec![], }); } @@ -57,11 +108,11 @@ fn main() -> Result<(), Error> { let cfg = config::load(std::path::Path::new(&source_path))?; iter::iter_crate(&source_path, |path, source| { - extractor::extract(&mut results, path, source) + extractor::extract(&mut results, path, source, cfg.attrs.clone()) })?; - if let Some(list) = args.tr { - add_translations(&list, &mut results); + if let Some(list) = args.translate { + add_translations(&list, &mut results, &cfg.attrs); } let mut messages: Vec<_> = results.iter().collect(); diff --git a/crates/extract/Cargo.toml b/crates/extract/Cargo.toml index f4522cd..fd83d8f 100644 --- a/crates/extract/Cargo.toml +++ b/crates/extract/Cargo.toml @@ -16,7 +16,7 @@ proc-macro2 = { version = "1", features = ["span-locations"] } quote = "1" regex = "1" rust-i18n-support = { path = "../support", version = "3.0.0" } -serde = "1" +serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.8" syn = { version = "2.0.18", features = ["full"] } diff --git a/crates/extract/src/attrs.rs b/crates/extract/src/attrs.rs new file mode 100644 index 0000000..eb8c954 --- /dev/null +++ b/crates/extract/src/attrs.rs @@ -0,0 +1,47 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct I18nAttrs { + #[serde(default = "minify_key")] + pub minify_key: bool, + #[serde(default = "minify_key_len")] + pub minify_key_len: usize, + #[serde(default = "minify_key_prefix")] + pub minify_key_prefix: String, + #[serde(default = "minify_key_thresh")] + pub minify_key_thresh: usize, +} + +impl I18nAttrs { + pub fn new() -> Self { + Self { + minify_key: rust_i18n_support::DEFAULT_MINIFY_KEY, + minify_key_len: rust_i18n_support::DEFAULT_MINIFY_KEY_LEN, + minify_key_prefix: rust_i18n_support::DEFAULT_MINIFY_KEY_PREFIX.to_string(), + minify_key_thresh: rust_i18n_support::DEFAULT_MINIFY_KEY_THRESH, + } + } +} + +impl Default for I18nAttrs { + fn default() -> Self { + Self::new() + } +} + +fn minify_key() -> bool { + I18nAttrs::default().minify_key +} + +fn minify_key_len() -> usize { + I18nAttrs::default().minify_key_len +} + +fn minify_key_prefix() -> String { + I18nAttrs::default().minify_key_prefix +} + +fn minify_key_thresh() -> usize { + I18nAttrs::default().minify_key_thresh +} diff --git a/crates/extract/src/extractor.rs b/crates/extract/src/extractor.rs index 2d1ba55..20e40d4 100644 --- a/crates/extract/src/extractor.rs +++ b/crates/extract/src/extractor.rs @@ -1,3 +1,4 @@ +use crate::attrs::I18nAttrs; use anyhow::Error; use proc_macro2::{TokenStream, TokenTree}; use quote::ToTokens; @@ -16,16 +17,16 @@ pub struct Location { pub struct Message { pub key: String, pub index: usize, - pub is_tr: bool, + pub minify_key: bool, pub locations: Vec, } impl Message { - fn new(key: &str, index: usize, is_tr: bool) -> Self { + fn new(key: &str, index: usize, minify_key: bool) -> Self { Self { key: key.to_owned(), index, - is_tr, + minify_key, locations: vec![], } } @@ -34,8 +35,17 @@ impl Message { static METHOD_NAMES: &[&str] = &["t", "tr"]; #[allow(clippy::ptr_arg)] -pub fn extract(results: &mut Results, path: &PathBuf, source: &str) -> Result<(), Error> { - let mut ex = Extractor { results, path }; +pub fn extract( + results: &mut Results, + path: &PathBuf, + source: &str, + attrs: I18nAttrs, +) -> Result<(), Error> { + let mut ex = Extractor { + results, + path, + attrs, + }; let file = syn::parse_file(source) .unwrap_or_else(|_| panic!("Failed to parse file, file: {}", path.display())); @@ -47,6 +57,7 @@ pub fn extract(results: &mut Results, path: &PathBuf, source: &str) -> Result<() struct Extractor<'a> { results: &'a mut Results, path: &'a PathBuf, + attrs: I18nAttrs, } impl<'a> Extractor<'a> { @@ -68,7 +79,7 @@ impl<'a> Extractor<'a> { let ident_str = ident.to_string(); if METHOD_NAMES.contains(&ident_str.as_str()) && is_macro { if let Some(TokenTree::Group(group)) = token_iter.peek() { - self.take_message(group.stream(), ident_str == "tr"); + self.take_message(group.stream()); } } } @@ -79,7 +90,7 @@ impl<'a> Extractor<'a> { Ok(()) } - fn take_message(&mut self, stream: TokenStream, is_tr: bool) { + fn take_message(&mut self, stream: TokenStream) { let mut token_iter = stream.into_iter().peekable(); let literal = if let Some(TokenTree::Literal(literal)) = token_iter.next() { @@ -88,13 +99,24 @@ impl<'a> Extractor<'a> { return; }; + let I18nAttrs { + minify_key, + minify_key_len, + minify_key_prefix, + minify_key_thresh, + } = &self.attrs; let key: Option = Some(literal); if let Some(lit) = key { if let Some(key) = literal_to_string(&lit) { - let (message_key, message_content) = if is_tr { - let hashed_key = rust_i18n_support::TrKey::tr_key(&key); - (hashed_key, key.clone()) + let (message_key, message_content) = if *minify_key { + let hashed_key = rust_i18n_support::MinifyKey::minify_key( + &key, + *minify_key_len, + minify_key_prefix, + *minify_key_thresh, + ); + (hashed_key.to_string(), key.clone()) } else { let message_key = format_message_key(&key); (message_key.clone(), message_key) @@ -103,7 +125,7 @@ impl<'a> Extractor<'a> { let message = self .results .entry(message_key) - .or_insert_with(|| Message::new(&message_content, index, is_tr)); + .or_insert_with(|| Message::new(&message_content, index, *minify_key)); let span = lit.span(); let line = span.start().line; @@ -151,7 +173,7 @@ mod tests { )+ ], index: 0, - is_tr: false, + minify_key: false, }; results.push(message); )+ @@ -220,6 +242,7 @@ mod tests { let mut ex = Extractor { results: &mut results, path: &"hello.rs".to_owned().into(), + attrs: I18nAttrs::default(), }; ex.invoke(stream).unwrap(); diff --git a/crates/extract/src/generator.rs b/crates/extract/src/generator.rs index fa1e253..db81e4f 100644 --- a/crates/extract/src/generator.rs +++ b/crates/extract/src/generator.rs @@ -85,14 +85,13 @@ fn generate_result<'a, P: AsRef>( } } - let key = if m.is_tr { key } else { &m.key }; if let Some(trs) = data.get(locale) { if trs.get(key).is_some() { continue; } } - let value = if m.is_tr { + let value = if m.minify_key { m.key.to_owned() } else { m.key.split('.').last().unwrap_or_default().to_string() diff --git a/crates/extract/src/lib.rs b/crates/extract/src/lib.rs index 9d03299..f55cb26 100644 --- a/crates/extract/src/lib.rs +++ b/crates/extract/src/lib.rs @@ -1,3 +1,4 @@ +pub mod attrs; pub mod extractor; pub mod generator; pub mod iter; From 2a9abe1e01b5396d629fc11686c4ab0504c5135f Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 14:12:17 +0800 Subject: [PATCH 27/39] Add `minify_key` benchmark and testing --- Cargo.toml | 4 + benches/bench.rs | 104 +------------------------ benches/minify_key.rs | 110 ++++++++++++++++++++++++++ tests/i18n_minify_key.rs | 74 ++++++++++++++++++ tests/integration_tests.rs | 154 +------------------------------------ tests/multi_threading.rs | 9 +-- 6 files changed, 194 insertions(+), 261 deletions(-) create mode 100644 benches/minify_key.rs create mode 100644 tests/i18n_minify_key.rs diff --git a/Cargo.toml b/Cargo.toml index bf3996a..0343a42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,5 +47,9 @@ members = [ harness = false name = "bench" +[[bench]] +harness = false +name = "minify_key" + [features] log-missing = ["rust-i18n-macro/log-missing"] diff --git a/benches/bench.rs b/benches/bench.rs index ac10f03..3e77d47 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -1,4 +1,4 @@ -use rust_i18n::{t, tr}; +use rust_i18n::t; rust_i18n::i18n!("./tests/locales"); @@ -79,108 +79,6 @@ fn bench_t(c: &mut Criterion) { ) }) }); - - c.bench_function("tr", |b| b.iter(|| tr!("hello"))); - - c.bench_function("tr_lorem_ipsum", |b| b.iter(|| tr!( - r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque sed nisi leo. Donec commodo in ex at aliquam. Nunc in aliquam arcu. Fusce mollis metus orci, ut sagittis erat lobortis sed. Morbi quis arcu ultrices turpis finibus tincidunt non in purus. Donec gravida condimentum sapien. Duis iaculis fermentum congue. Quisque blandit libero a lacus auctor vestibulum. Nunc efficitur sollicitudin nisi, sit amet tristique lectus mollis non. Praesent sit amet erat volutpat, pharetra orci eget, rutrum felis. Sed elit augue, imperdiet eu facilisis vel, finibus vel urna. Duis quis neque metus. - - Mauris suscipit bibendum mattis. Vestibulum eu augue diam. Morbi dapibus tempus viverra. Sed aliquam turpis eget justo ornare maximus vitae et tortor. Donec semper neque sit amet sapien congue scelerisque. Maecenas bibendum imperdiet dolor interdum facilisis. Integer non diam tempus, pharetra ex at, euismod diam. Ut enim turpis, sagittis in iaculis ut, finibus et sem. Suspendisse a felis euismod neque euismod placerat. Praesent ipsum libero, porta vel egestas quis, aliquet vitae lorem. Nullam vel pharetra erat, sit amet sodales leo."# - ))); - - c.bench_function("tr_with_locale", |b| b.iter(|| tr!("hello", locale = "en"))); - - c.bench_function("tr_with_threads", |b| { - let exit_loop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let mut handles = Vec::new(); - for _ in 0..4 { - let exit_loop = exit_loop.clone(); - handles.push(std::thread::spawn(move || { - while !exit_loop.load(std::sync::atomic::Ordering::SeqCst) { - criterion::black_box(tr!("hello")); - } - })); - } - b.iter(|| tr!("hello")); - exit_loop.store(true, std::sync::atomic::Ordering::SeqCst); - for handle in handles { - handle.join().unwrap(); - } - }); - - c.bench_function("tr_with_args", |b| { - b.iter(|| { - tr!( - "Hello, %{name}. Your message is: %{msg}", - name = "Jason", - msg = "Bla bla" - ) - }) - }); - - c.bench_function("tr_with_args (str)", |b| { - b.iter(|| { - tr!( - "Hello, %{name}. Your message is: %{msg}", - "name" = "Jason", - "msg" = "Bla bla" - ) - }) - }); - - c.bench_function("tr_with_args (many)", |b| { - b.iter(|| { - tr!( - r#"Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. - You live in %{city} %{zip}. - Your website is %{website}."#, - id = 123, - name = "Marion", - surname = "Christiansen", - email = "Marion_Christiansen83@hotmail.com", - city = "Litteltown", - zip = 8408, - website = "https://snoopy-napkin.name" - ) - }) - }); - - c.bench_function("tr_with_args (many-dynamic)", |b| { - let msg = - r#"Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. - You live in %{city} %{zip}. - Your website is %{website}."# - .to_string(); - b.iter(|| { - tr!( - &msg, - id = 123, - name = "Marion", - surname = "Christiansen", - email = "Marion_Christiansen83@hotmail.com", - city = "Litteltown", - zip = 8408, - website = "https://snoopy-napkin.name" - ) - }) - }); - - c.bench_function("format! (many)", |b| { - b.iter(|| { - format!( - r#"Hello {name} %{surname}, your account id is {id}, email address is {email}. - You live in {city} {zip}. - Your website is {website}."#, - id = 123, - name = "Marion", - surname = "Christiansen", - email = "Marion_Christiansen83@hotmail.com", - city = "Litteltown", - zip = 8408, - website = "https://snoopy-napkin.name" - ); - }) - }); } criterion_group!(benches, bench_t); diff --git a/benches/minify_key.rs b/benches/minify_key.rs new file mode 100644 index 0000000..3e0425f --- /dev/null +++ b/benches/minify_key.rs @@ -0,0 +1,110 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use rust_i18n::t; + +rust_i18n::i18n!("./tests/locales", minify_key = true, minify_key_len = 12); + +pub fn bench_t(c: &mut Criterion) { + c.bench_function("t", |b| b.iter(|| t!("hello"))); + + c.bench_function("t_lorem_ipsum", |b| b.iter(|| t!( + r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque sed nisi leo. Donec commodo in ex at aliquam. Nunc in aliquam arcu. Fusce mollis metus orci, ut sagittis erat lobortis sed. Morbi quis arcu ultrices turpis finibus tincidunt non in purus. Donec gravida condimentum sapien. Duis iaculis fermentum congue. Quisque blandit libero a lacus auctor vestibulum. Nunc efficitur sollicitudin nisi, sit amet tristique lectus mollis non. Praesent sit amet erat volutpat, pharetra orci eget, rutrum felis. Sed elit augue, imperdiet eu facilisis vel, finibus vel urna. Duis quis neque metus. + + Mauris suscipit bibendum mattis. Vestibulum eu augue diam. Morbi dapibus tempus viverra. Sed aliquam turpis eget justo ornare maximus vitae et tortor. Donec semper neque sit amet sapien congue scelerisque. Maecenas bibendum imperdiet dolor interdum facilisis. Integer non diam tempus, pharetra ex at, euismod diam. Ut enim turpis, sagittis in iaculis ut, finibus et sem. Suspendisse a felis euismod neque euismod placerat. Praesent ipsum libero, porta vel egestas quis, aliquet vitae lorem. Nullam vel pharetra erat, sit amet sodales leo."# + ))); + + c.bench_function("t_with_locale", |b| b.iter(|| t!("hello", locale = "en"))); + + c.bench_function("tr_with_threads", |b| { + let exit_loop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mut handles = Vec::new(); + for _ in 0..4 { + let exit_loop = exit_loop.clone(); + handles.push(std::thread::spawn(move || { + while !exit_loop.load(std::sync::atomic::Ordering::SeqCst) { + criterion::black_box(t!("hello")); + } + })); + } + b.iter(|| t!("hello")); + exit_loop.store(true, std::sync::atomic::Ordering::SeqCst); + for handle in handles { + handle.join().unwrap(); + } + }); + + c.bench_function("tr_with_args", |b| { + b.iter(|| { + t!( + "Hello, %{name}. Your message is: %{msg}", + name = "Jason", + msg = "Bla bla" + ) + }) + }); + + c.bench_function("tr_with_args (str)", |b| { + b.iter(|| { + t!( + "Hello, %{name}. Your message is: %{msg}", + "name" = "Jason", + "msg" = "Bla bla" + ) + }) + }); + + c.bench_function("tr_with_args (many)", |b| { + b.iter(|| { + t!( + r#"Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. + You live in %{city} %{zip}. + Your website is %{website}."#, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ) + }) + }); + + c.bench_function("t_with_args (many-dynamic)", |b| { + let msg = r#"Hello %{name} %{surname}, your account id is %{id}, email address is %{email}. + You live in %{city} %{zip}. + Your website is %{website}."# + .to_string(); + b.iter(|| { + t!( + &msg, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ) + }) + }); + + c.bench_function("format! (many)", |b| { + b.iter(|| { + format!( + r#"Hello {name} %{surname}, your account id is {id}, email address is {email}. + You live in {city} {zip}. + Your website is {website}."#, + id = 123, + name = "Marion", + surname = "Christiansen", + email = "Marion_Christiansen83@hotmail.com", + city = "Litteltown", + zip = 8408, + website = "https://snoopy-napkin.name" + ); + }) + }); +} + +criterion_group!(benches, bench_t); +criterion_main!(benches); diff --git a/tests/i18n_minify_key.rs b/tests/i18n_minify_key.rs new file mode 100644 index 0000000..f25d6cc --- /dev/null +++ b/tests/i18n_minify_key.rs @@ -0,0 +1,74 @@ +rust_i18n::i18n!( + "./tests/locales", + fallback = "en", + minify_key = true, + minify_key_len = 24, + minify_key_prefix = "tr_", + minify_key_thresh = 4 +); + +#[cfg(test)] +mod tests { + use super::*; + use rust_i18n::{t, tkv}; + + #[test] + fn test_i18n_attrs() { + assert_eq!(crate::_RUST_I18N_MINIFY_KEY, true); + assert_eq!(crate::_RUST_I18N_MINIFY_KEY_LEN, 24); + assert_eq!(crate::_RUST_I18N_MINIFY_KEY_PREFIX, "tr_"); + assert_eq!(crate::_RUST_I18N_MINIFY_KEY_THRESH, 4); + } + + #[test] + fn test_t() { + assert_eq!(t!("Bar - Hello, World!"), "Bar - Hello, World!"); + assert_eq!( + t!("Bar - Hello, World!", locale = "en"), + "Bar - Hello, World!" + ); + assert_eq!( + t!("Bar - Hello, World!", locale = "zh-CN"), + "Bar - 你好世界!" + ); + let fruits = vec!["Apple", "Banana", "Orange"]; + let fruits_translated = vec!["苹果", "香蕉", "橘子"]; + for (src, dst) in fruits.iter().zip(fruits_translated.iter()) { + assert_eq!(t!(*src, locale = "zh-CN"), *dst); + } + let msg = "aka".to_string(); + let i = 0; + assert_eq!(t!(msg, name => & i : {} ), "aka"); + assert_eq!(t!("hello"), "hello"); + assert_eq!(t!("hello",), "hello"); + assert_eq!(t!("hello", locale = "en"), "hello"); + assert_eq!(t!(format!("hello"), locale = "en"), "hello"); + assert_eq!(t!("Hello, %{name}", name = "Bar"), "Hello, Bar"); + assert_eq!( + t!("You have %{count} messages.", locale = "zh-CN", count = 1 + 2,,,), + "你收到了 3 条新消息。" + ); + let (key, msg) = tkv!( + r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque sed nisi leo. Donec commodo in ex at aliquam. Nunc in aliquam arcu. Fusce mollis metus orci, ut sagittis erat lobortis sed. Morbi quis arcu ultrices turpis finibus tincidunt non in purus. Donec gravida condimentum sapien. Duis iaculis fermentum congue. Quisque blandit libero a lacus auctor vestibulum. Nunc efficitur sollicitudin nisi, sit amet tristique lectus mollis non. Praesent sit amet erat volutpat, pharetra orci eget, rutrum felis. Sed elit augue, imperdiet eu facilisis vel, finibus vel urna. Duis quis neque metus. + + Mauris suscipit bibendum mattis. Vestibulum eu augue diam. Morbi dapibus tempus viverra. Sed aliquam turpis eget justo ornare maximus vitae et tortor. Donec semper neque sit amet sapien congue scelerisque. Maecenas bibendum imperdiet dolor interdum facilisis. Integer non diam tempus, pharetra ex at, euismod diam. Ut enim turpis, sagittis in iaculis ut, finibus et sem. Suspendisse a felis euismod neque euismod placerat. Praesent ipsum libero, porta vel egestas quis, aliquet vitae lorem. Nullam vel pharetra erat, sit amet sodales leo."# + ); + assert_eq!(t!((key, msg)).as_ptr(), msg.as_ptr()); + assert_eq!(t!((key, msg), locale = "en"), msg); + assert_eq!(t!((key, msg), locale = "de"), msg); + assert_eq!(t!((key, msg), locale = "zh"), msg); + } + + #[test] + fn test_tkv() { + let (key, msg) = tkv!(""); + assert_eq!(key, ""); + assert_eq!(msg, ""); + let (key, msg) = tkv!("Hey"); + assert_eq!(key, "Hey"); + assert_eq!(msg, "Hey"); + let (key, msg) = tkv!("Hello, world!"); + assert_eq!(key, "tr_1LokVzuiIrh1xByyZG4wjZ"); + assert_eq!(msg, "Hello, world!"); + } +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4fe636d..e8d86c2 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -33,7 +33,7 @@ rust_i18n::i18n!( #[cfg(test)] mod tests { - use rust_i18n::{t, tr}; + use rust_i18n::t; use rust_i18n_support::load_locales; mod test0 { @@ -201,158 +201,6 @@ mod tests { ); } - #[test] - fn test_tr() { - rust_i18n::set_locale("en"); - assert_eq!(tr!("Bar - Hello, World!"), "Bar - Hello, World!"); - - // Vars - assert_eq!( - tr!("Hello, %{name}. Your message is: %{msg}"), - "Hello, %{name}. Your message is: %{msg}" - ); - assert_eq!( - tr!("Hello, %{name}. Your message is: %{msg}", name = "Jason"), - "Hello, Jason. Your message is: %{msg}" - ); - assert_eq!( - tr!( - "Hello, %{name}. Your message is: %{msg}", - name = "Jason", - msg = "Bla bla" - ), - "Hello, Jason. Your message is: Bla bla" - ); - - rust_i18n::set_locale("zh-CN"); - assert_eq!(tr!("Hello, %{name}!", name => "world"), "你好,world!"); - assert_eq!(tr!("Hello, %{name}!", name = "world"), "你好,world!"); - - rust_i18n::set_locale("en"); - assert_eq!(tr!("Hello, %{name}!", name = "world"), "Hello, world!"); - - let fruits = vec!["Apple", "Banana", "Orange"]; - let fruits_translated = vec!["苹果", "香蕉", "橘子"]; - for (src, dst) in fruits.iter().zip(fruits_translated.iter()) { - assert_eq!(tr!(*src, locale = "zh-CN"), *dst); - } - } - - #[test] - fn test_tr_with_tt_val() { - rust_i18n::set_locale("en"); - - assert_eq!( - tr!("You have %{count} messages.", count = 100), - "You have 100 messages." - ); - assert_eq!( - tr!("You have %{count} messages.", count = 1.01), - "You have 1.01 messages." - ); - assert_eq!( - tr!("You have %{count} messages.", count = 1 + 2), - "You have 3 messages." - ); - - // Test end with a comma - assert_eq!( - tr!( - "You have %{count} messages.", - locale = "zh-CN", - count = 1 + 2, - ), - "你收到了 3 条新消息。" - ); - - let a = 100; - assert_eq!( - tr!("You have %{count} messages.", count = a / 2), - "You have 50 messages." - ); - } - - #[test] - fn test_tr_with_locale_and_args() { - rust_i18n::set_locale("en"); - - assert_eq!( - tr!("Bar - Hello, World!", locale = "zh-CN"), - "Bar - 你好世界!" - ); - assert_eq!( - tr!("Bar - Hello, World!", locale = "en"), - "Bar - Hello, World!" - ); - - assert_eq!(tr!("Hello, %{name}!", name = "Jason"), "Hello, Jason!"); - assert_eq!( - tr!("Hello, %{name}!", locale = "en", name = "Jason"), - "Hello, Jason!" - ); - // Invalid locale position, will ignore - assert_eq!( - tr!("Hello, %{name}!", name = "Jason", locale = "en"), - "Hello, Jason!" - ); - assert_eq!( - tr!("Hello, %{name}!", locale = "zh-CN", name = "Jason"), - "你好,Jason!" - ); - } - - #[test] - fn test_tr_with_hash_args() { - rust_i18n::set_locale("en"); - - // Hash args - assert_eq!(tr!("Hello, %{name}!", name => "Jason"), "Hello, Jason!"); - assert_eq!(tr!("Hello, %{name}!", "name" => "Jason"), "Hello, Jason!"); - assert_eq!( - tr!("Hello, %{name}!", locale = "zh-CN", "name" => "Jason"), - "你好,Jason!" - ); - } - - #[test] - fn test_tr_with_specified_args() { - #[derive(Debug)] - struct Foo { - #[allow(unused)] - bar: usize, - } - assert_eq!( - tr!("Any: %{value}", value = Foo { bar : 1 } : {:?}), - "Any: Foo { bar: 1 }" - ); - let foo = Foo { bar: 2 }; - assert_eq!( - tr!("Any: %{value}", value = foo : {:?}), - "Any: Foo { bar: 2 }" - ); - assert_eq!( - tr!("Any: %{value}", value = &foo : {:?}), - "Any: Foo { bar: 2 }" - ); - assert_eq!(tr!("Any: %{value}", value = foo.bar : {:?}), "Any: 2"); - assert_eq!( - tr!("You have %{count} messages.", count => 123 : {:08}), - "You have 00000123 messages." - ); - assert_eq!( - tr!("You have %{count} messages.", count => 100 + 23 : {:>8}), - "You have 123 messages." - ); - assert_eq!( - tr!("You have %{count} messages.", count => 1 * 100 + 23 : {:08}, locale = "zh-CN"), - "你收到了 00000123 条新消息。" - ); - assert_eq!( - tr!("You have %{count} messages.", count => 100 + 23 * 1 / 1 : {:>8}, locale = "zh-CN"), - "你收到了 123 条新消息。" - ); - } - #[test] fn test_with_merge_file() { rust_i18n::set_locale("en"); diff --git a/tests/multi_threading.rs b/tests/multi_threading.rs index d10e959..3afb96d 100644 --- a/tests/multi_threading.rs +++ b/tests/multi_threading.rs @@ -1,9 +1,8 @@ +use rust_i18n::{set_locale, t}; use std::ops::Add; use std::thread::spawn; use std::time::{Duration, Instant}; -use rust_i18n::{set_locale, t, tr}; - rust_i18n::i18n!("locales", fallback = "en"); #[test] @@ -34,7 +33,7 @@ fn test_load_and_store() { } #[test] -fn test_tr_concurrent() { +fn test_t_concurrent() { let end = Instant::now().add(Duration::from_secs(3)); let store = spawn(move || { let mut i = 0u32; @@ -58,9 +57,9 @@ fn test_tr_concurrent() { for i in 0..100usize { let m = i.checked_rem(num_locales).unwrap_or_default(); if m == 0 { - tr!("hello"); + t!("hello"); } else { - tr!("hello", locale = locales[m]); + t!("hello", locale = locales[m]); } } } From 1fa8630e18b7beb2f408bf77da67f4a07a3dc70c Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 14:25:28 +0800 Subject: [PATCH 28/39] Add new example: app-minify-key --- Cargo.toml | 2 +- .../{app-tr => app-minify-key}/Cargo.toml | 4 +- .../{app-tr => app-minify-key}/locales/v2.yml | 125 +----------------- examples/app-minify-key/src/main.rs | 87 ++++++++++++ examples/app-tr/src/main.rs | 97 -------------- 5 files changed, 97 insertions(+), 218 deletions(-) rename examples/{app-tr => app-minify-key}/Cargo.toml (81%) rename examples/{app-tr => app-minify-key}/locales/v2.yml (57%) create mode 100644 examples/app-minify-key/src/main.rs delete mode 100644 examples/app-tr/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 0343a42..b100f13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ members = [ "crates/support", "crates/macro", "examples/app-load-path", - "examples/app-tr", + "examples/app-minify-key", "examples/foo", ] diff --git a/examples/app-tr/Cargo.toml b/examples/app-minify-key/Cargo.toml similarity index 81% rename from examples/app-tr/Cargo.toml rename to examples/app-minify-key/Cargo.toml index b812f17..952e9c2 100644 --- a/examples/app-tr/Cargo.toml +++ b/examples/app-minify-key/Cargo.toml @@ -1,6 +1,6 @@ [package] edition = "2021" -name = "app-tr" +name = "app-minify-key" version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -11,7 +11,7 @@ log = { version = "0.4", optional = true } rust-i18n = { path = "../.." } [features] -log-tr-dyn = ["env_logger", "log", "rust-i18n/log-tr-dyn"] +log-missing = ["env_logger", "log", "rust-i18n/log-missing"] [package.metadata.i18n] available-locales = ["en", "zh-CN"] diff --git a/examples/app-tr/locales/v2.yml b/examples/app-minify-key/locales/v2.yml similarity index 57% rename from examples/app-tr/locales/v2.yml rename to examples/app-minify-key/locales/v2.yml index 063d422..db97dfd 100644 --- a/examples/app-tr/locales/v2.yml +++ b/examples/app-minify-key/locales/v2.yml @@ -1,5 +1,5 @@ _version: 2 -tr_29xGXAUPAkgvVzCf9ES3q8: +T.29xGXAUPAkgvVzCf9ES3q8: en: Apple de: Apfel fr: Pomme @@ -10,7 +10,7 @@ tr_29xGXAUPAkgvVzCf9ES3q8: vi: Táo zh: 苹果 zh-TW: 蘋果 -tr_7MQVq9vgi0h6pLE47CdWXH: +T.7MQVq9vgi0h6pLE47CdWXH: en: Banana de: Banane fr: Banane @@ -21,7 +21,7 @@ tr_7MQVq9vgi0h6pLE47CdWXH: vi: Chuối zh: 香蕉 zh-TW: 香蕉 -tr_2RohljPx99sA18L8E5oTD4: +T.2RohljPx99sA18L8E5oTD4: en: Orange de: Orange fr: Orange @@ -32,7 +32,7 @@ tr_2RohljPx99sA18L8E5oTD4: vi: Cam zh: 橘子 zh-TW: 橘子 -tr_KtVLkBzuyfHoJZtCWEIOU: +T.KtVLkBzuyfHoJZtCWEIOU: en: Hello de: Hallo fr: Bonjour @@ -42,9 +42,8 @@ tr_KtVLkBzuyfHoJZtCWEIOU: ru: Привет vi: Xin chào zh: 你好 - zh: 你好 zh-TW: 妳好 -tr_tvpNzQjFwc0trHvgzSBxX: +T.tvpNzQjFwc0trHvgzSBxX: en: Hello, %{name}! de: Hallo, %{name}! fr: Bonjour, %{name}! @@ -55,7 +54,7 @@ tr_tvpNzQjFwc0trHvgzSBxX: vi: Xin chào, %{name}! zh: 你好,%{name}! zh-TW: 妳好,%{name}! -tr_kmFrQ2nnJsvUh3Ckxmki0: +T.kmFrQ2nnJsvUh3Ckxmki0: en: "Hello, %{name}. Your message is: %{msg}" de: "Hallo, %{name}. Deine Nachricht ist: %{msg}" fr: "Bonjour, %{name}. Votre message est: %{msg}" @@ -66,7 +65,7 @@ tr_kmFrQ2nnJsvUh3Ckxmki0: vi: "Xin chào, %{name}. Tin nhắn của bạn là: %{msg}" zh: "你好,%{name}。这是你的消息:%{msg}" zh-TW: "妳好,%{name}。這是妳的消息:%{msg}" -tr_7hWbTwvMpr0H0oDLIQlfrm: +T.7hWbTwvMpr0H0oDLIQlfrm: en: You have %{count} messages. de: Du hast %{count} Nachrichten. fr: Vous avez %{count} messages. @@ -77,113 +76,3 @@ tr_7hWbTwvMpr0H0oDLIQlfrm: vi: Bạn có %{count} tin nhắn. zh: 你收到了 %{count} 条新消息。 zh-TW: 妳收到了 %{count} 條新消息。 -tr_N_0: - en: Zero - de: Null - fr: Zéro - it: Zero - ja: ゼロ - ko: 영 - ru: Ноль - vi: Không - zh: 零 - zh-TW: 零 -tr_N_1: - en: One - de: Eins - fr: Un - it: Uno - ja: 一 - ko: 일 - ru: Один - vi: Một - zh: 一 - zh-TW: 一 -tr_N_2: - en: Two - de: Zwei - fr: Deux - it: Due - ja: 二 - ko: 이 - ru: Два - vi: Hai - zh: 二 - zh-TW: 二 -tr_N_3: - en: Three - de: Drei - fr: Trois - it: Tre - ja: 三 - ko: 삼 - ru: Три - vi: Ba - zh: 三 - zh-TW: 三 -tr_N_4: - en: Four - de: Vier - fr: Quatre - it: Quattro - ja: 四 - ko: 사 - ru: Четыре - vi: Bốn - zh: 四 - zh-TW: 四 -tr_N_5: - en: Five - de: Fünf - fr: Cinq - it: Cinque - ja: 五 - ko: 오 - ru: Пять - vi: Năm - zh: 五 - zh-TW: 五 -tr_N_6: - en: Six - de: Sechs - fr: Six - it: Sei - ja: 六 - ko: 육 - ru: Шесть - vi: Sáu - zh: 六 - zh-TW: 六 -tr_N_7: - en: Seven - de: Sieben - fr: Sept - it: Sette - ja: 七 - ko: 칠 - ru: Семь - vi: Bảy - zh: 七 - zh-TW: 七 -tr_N_8: - en: Eight - de: Acht - fr: Huit - it: Otto - ja: 八 - ko: 팔 - ru: Восемь - vi: Tám - zh: 八 - zh-TW: 八 -tr_N_9: - en: Nine - de: Neun - fr: Neuf - it: Nove - ja: 九 - ko: 구 - ru: Девять - vi: Chín - zh: 九 - zh-TW: 九 diff --git a/examples/app-minify-key/src/main.rs b/examples/app-minify-key/src/main.rs new file mode 100644 index 0000000..2091519 --- /dev/null +++ b/examples/app-minify-key/src/main.rs @@ -0,0 +1,87 @@ +use rust_i18n::t; + +rust_i18n::i18n!( + "locales", + minify_key = true, + minify_key_len = 24, + minify_key_prefix = "T.", + minify_key_thresh = 4 +); + +#[cfg(feature = "log-missing")] +fn set_logger() { + env_logger::builder() + .filter_level(log::LevelFilter::Debug) + .parse_default_env() + .init(); +} + +#[cfg(not(feature = "log-missing"))] +fn set_logger() {} + +fn main() { + set_logger(); + + let locales = rust_i18n::available_locales!(); + println!("Available locales: {:?}", locales); + println!(); + + println!("Translation of string literals:"); + for locale in &locales { + println!( + "{:>8} => {} ({})", + "Hello", + t!("Hello", locale = locale), + locale + ); + } + println!(); + + println!("Translation of string literals with patterns:"); + for locale in &locales { + println!( + "Hello, %{{name}}! => {} ({})", + t!("Hello, %{name}!", name = "World", locale = locale), + locale + ); + } + println!(); + + println!("Translation of string literals with specified arguments:"); + for i in (0..10000).step_by(50) { + println!( + "Zero padded number: %{{count}} => {}", + t!("Zero padded number: %{count}", count = i : {:08}), + ); + } + println!(); + + println!("Handling of missing translations:"); + for locale in &locales { + println!( + "{:>8} => {} ({locale})", + "The message is untranslated!", + t!("The message is untranslated!", locale = locale) + ); + } + println!(); + + println!("Translation of runtime strings:"); + let src_list = ["Apple", "Banana", "Orange"]; + for src in src_list.iter() { + for locale in &locales { + let translated = t!(*src, locale = locale); + println!("{:>8} => {} ({locale})", src, translated); + } + } + println!(); + + if cfg!(feature = "log-missing") { + println!("Translates runtime strings and logs when a lookup is missing:"); + for locale in &locales { + let msg = "Foo Bar".to_string(); + println!("{:>8} => {} ({locale})", &msg, t!(&msg, locale = locale)); + } + println!(); + } +} diff --git a/examples/app-tr/src/main.rs b/examples/app-tr/src/main.rs deleted file mode 100644 index 58e2622..0000000 --- a/examples/app-tr/src/main.rs +++ /dev/null @@ -1,97 +0,0 @@ -use rust_i18n::tr; - -rust_i18n::i18n!("locales"); - -#[cfg(feature = "log-tr-dyn")] -fn set_logger() { - env_logger::builder() - .filter_level(log::LevelFilter::Debug) - .parse_default_env() - .init(); -} - -#[cfg(not(feature = "log-tr-dyn"))] -fn set_logger() {} - -fn main() { - set_logger(); - - let locales = rust_i18n::available_locales!(); - println!("Available locales: {:?}", locales); - println!(); - - println!("String literals with patterns translation:"); - for locale in &locales { - println!( - "Hello, %{{name}}! => {} ({})", - tr!("Hello, %{name}!", name = "World", locale = locale), - locale - ); - } - println!(); - - println!("String literals translation:"); - for locale in &locales { - println!( - "{:>8} => {} ({})", - "Hello", - tr!("Hello", locale = locale), - locale - ); - } - println!(); - - // Identify the locale message by number. - // For example, the `tr!` will find the translation with key named "tr_N_5" for 5. - println!("Numeric literals translation:"); - for locale in &locales { - println!("{:>8} => {} ({})", 5, tr!(5, locale = locale), locale); - } - println!(); - - println!("Missing translations:"); - for locale in &locales { - println!( - "{:>8} => {} ({locale})", - "The message is untranslated!", - tr!("The message is untranslated!", locale = locale) - ); - } - println!(); - - println!("Runtime string translation:"); - let src_list = ["Apple", "Banana", "Orange"]; - for src in src_list.iter() { - for locale in &locales { - let translated = tr!(*src, locale = locale); - println!("{:>8} => {} ({locale})", src, translated); - } - } - println!(); - - println!("Runtime numeric translation:"); - for i in 0..10usize { - for locale in &locales { - println!("{:>8} => {} ({locale})", i, tr!(i, locale = locale)); - } - } - println!(); - - if cfg!(feature = "log-tr-dyn") { - println!("Runtime string missing with logging:"); - for locale in &locales { - let msg = "Foo Bar".to_string(); - println!("{:>8} => {} ({locale})", &msg, tr!(&msg, locale = locale)); - } - println!(); - } - - println!("String literals with specified args translation:"); - for i in (0..10000).step_by(50) { - println!( - "Zero padded number: %{{count}} => {}", - tr!("Zero padded number: %{count}", count = i : {:08}), - ); - } - println!(); -} From 1b19e83ccbe960b2b382f0158c44b2177a57a366 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 15:14:56 +0800 Subject: [PATCH 29/39] Add new example: app-egui --- Cargo.toml | 1 + examples/app-egui/Cargo.toml | 24 +++++++ examples/app-egui/locales/v2.yml | 52 +++++++++++++++ examples/app-egui/src/main.rs | 111 +++++++++++++++++++++++++++++++ 4 files changed, 188 insertions(+) create mode 100644 examples/app-egui/Cargo.toml create mode 100644 examples/app-egui/locales/v2.yml create mode 100644 examples/app-egui/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index b100f13..0303467 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "crates/extract", "crates/support", "crates/macro", + "examples/app-egui", "examples/app-load-path", "examples/app-minify-key", "examples/foo", diff --git a/examples/app-egui/Cargo.toml b/examples/app-egui/Cargo.toml new file mode 100644 index 0000000..a046a7a --- /dev/null +++ b/examples/app-egui/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "app-egui" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +eframe = "0.25.0" +env_logger = { version = "0.11", optional = true } +log = { version = "0.4", optional = true } +rust-i18n = { path = "../.." } + +[features] +default = ["log-missing"] +log-missing = ["env_logger", "log", "rust-i18n/log-missing"] + +[package.metadata.i18n] +available-locales = ["en", "fr", "ja", "ko", "zh", "zh-CN"] +default-locale = "en" +minify-key = true +minify-key-len = 12 +minify-key-prefix = "T." +minify-key-thresh = 8 diff --git a/examples/app-egui/locales/v2.yml b/examples/app-egui/locales/v2.yml new file mode 100644 index 0000000..3e37ef0 --- /dev/null +++ b/examples/app-egui/locales/v2.yml @@ -0,0 +1,52 @@ +_version: 2 + +Arthur: + en: Arthur + fr: Arthur + ja: アーサー + ko: 아서 + zh: 亚瑟 + zh-TW: 亞瑟 +T.3UjfzBSnm1Qc: + en: My egui App + fr: Mon application egui + ja: 私の EGUI アプリ + ko: 내 egui 앱 + zh: 我的 EGUI 应用 + zh-TW: 我的 EGUI 應用 +T.4E2YxmZoydHX: + en: Click each year + fr: Cliquez sur chaque année + ja: 各年をクリック + ko: 각 연도를 클릭 + zh: 点击大一岁 + zh-TW: 點擊大一歲 +T.5o7tEO3Df7z8: + en: "Hello '%{name}', age %{age}" + fr: "Bonjour '%{name}', âge %{age}" + ja: "こんにちは '%{name}', 年齢 %{age}" + ko: "안녕하세요 '%{name}', 나이 %{age}" + zh: "你好 “%{name}”, 年龄 %{age}" + zh-TW: "妳好 “%{name}”, 年齡 %{age}" +T.6JOKobVbFAxv: + en: My egui Application + fr: Mon application egui + ja: 私の EGUI アプリケーション + ko: 내 egui 애플리케이션 + zh: 我的 EGUI 应用程序 + zh-TW: 我的 EGUI 應用程式 +T.gOXpNwPJI1fQ: + en: "Your name: " + fr: "Votre nom : " + ja: "あなたの名前:" + ko: "당신의 이름: " + zh: "你的名字:" + zh-TW: "你的名字:" +age: + en: age + fr: âge + ja: 年齢 + ko: 나이 + zh: 岁 + zh-TW: 歲 + diff --git a/examples/app-egui/src/main.rs b/examples/app-egui/src/main.rs new file mode 100644 index 0000000..4462942 --- /dev/null +++ b/examples/app-egui/src/main.rs @@ -0,0 +1,111 @@ +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] // hide console window on Windows in release + +use eframe::egui::{self, TextBuffer}; +use rust_i18n::t; + +rust_i18n::i18n!( + "locales", + minify_key = true, + minify_key_len = 12, + minify_key_prefix = "T.", + minify_key_thresh = 8 +); + +fn main() -> Result<(), eframe::Error> { + env_logger::init(); // Log to stderr (if you run with `RUST_LOG=debug`). + let options = eframe::NativeOptions { + viewport: egui::ViewportBuilder::default().with_inner_size([320.0, 240.0]), + ..Default::default() + }; + eframe::run_native( + t!("My egui App").as_str(), + options, + Box::new(|cc| { + // This gives us image support: + // egui_extras::install_image_loaders(&cc.egui_ctx); + setup_custom_fonts(&cc.egui_ctx); + Box::::default() + }), + ) +} + +struct MyApp { + name: String, + age: u32, + locale_id: usize, +} + +impl Default for MyApp { + fn default() -> Self { + Self { + name: t!("Arthur").into(), + age: 42, + locale_id: 0, + } + } +} + +impl eframe::App for MyApp { + fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) { + egui::CentralPanel::default().show(ctx, |ui| { + ui.heading(t!("My egui Application")); + ui.horizontal(|ui| { + let name_label = ui.label(t!("Your name: ")); + ui.text_edit_singleline(&mut self.name) + .labelled_by(name_label.id); + }); + ui.add(egui::Slider::new(&mut self.age, 0..=120).text(t!("age"))); + if ui.button(t!("Click each year")).clicked() { + self.age += 1; + } + ui.label(t!("Hello '%{name}', age %{age}", name => self.name, age => self.age)); + + ui.separator(); + + ui.horizontal(|ui| { + let locales = rust_i18n::available_locales!(); + for (i, locale) in locales.iter().enumerate() { + if ui + .selectable_value(&mut self.locale_id, i, *locale) + .changed() + { + rust_i18n::set_locale(locale); + ui.ctx().send_viewport_cmd(egui::ViewportCommand::Title( + t!("My egui App").to_string(), + )); + } + } + }); + }); + } +} + +fn setup_custom_fonts(ctx: &egui::Context) { + // Start with the default fonts (we will be adding to them rather than replacing them). + let mut fonts = egui::FontDefinitions::default(); + + // Install my own font (maybe supporting non-latin characters). + // .ttf and .otf files supported. + // NOTE: only support Windows and Simplified Chinese for now. + fonts.font_data.insert( + "my_font".to_owned(), + egui::FontData::from_static(include_bytes!("C:/Windows/Fonts/msyh.ttc")), + ); + + // Put my font first (highest priority) for proportional text: + fonts + .families + .entry(egui::FontFamily::Proportional) + .or_default() + .insert(0, "my_font".to_owned()); + + // Put my font as last fallback for monospace: + fonts + .families + .entry(egui::FontFamily::Monospace) + .or_default() + .push("my_font".to_owned()); + + // Tell egui to use these fonts: + ctx.set_fonts(fonts); +} From 6cfbef7f03b92fdd07a3a3c33da9af21173521fc Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 15:26:11 +0800 Subject: [PATCH 30/39] Move rust-i18n-cli argument `SOURCE` to last --- crates/cli/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 295f807..1d75711 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -27,9 +27,6 @@ enum CargoCli { /// /// https://github.com/longbridgeapp/rust-i18n struct I18nArgs { - /// Extract all untranslated I18n texts from source code - #[arg(default_value = "./")] - source: Option, /// Manually add a translation to the localization file. /// /// This is useful for non-literal values in the `t!` macro. @@ -41,6 +38,9 @@ struct I18nArgs { /// NOTE: The whitespace before and after the key and value will be trimmed. #[arg(short, long, default_value = None, name = "TEXT", num_args(1..), value_parser = translate_value_parser, verbatim_doc_comment)] translate: Option>, + /// Extract all untranslated I18n texts from source code + #[arg(default_value = "./", last = true)] + source: Option, } /// Remove quotes from a string at the start and end. From ee998bd9160da755ed765d5a6b798e247a809abe Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 15:31:21 +0800 Subject: [PATCH 31/39] Cleanup clippy warnings --- build.rs | 2 +- crates/support/src/backend.rs | 6 ++++++ crates/support/src/lib.rs | 12 +++++------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/build.rs b/build.rs index 063b9be..8e65a2b 100644 --- a/build.rs +++ b/build.rs @@ -25,7 +25,7 @@ fn main() { let workdir = workdir().unwrap_or("./".to_string()); let locale_path = format!("{workdir}/**/locales/**/*"); - if let Ok(globs) = globwalk::glob(&locale_path) { + if let Ok(globs) = globwalk::glob(locale_path) { for entry in globs { if let Err(e) = entry { println!("cargo:i18n-error={}", e); diff --git a/crates/support/src/backend.rs b/crates/support/src/backend.rs index cbea62c..c3ead73 100644 --- a/crates/support/src/backend.rs +++ b/crates/support/src/backend.rs @@ -105,6 +105,12 @@ impl Backend for SimpleBackend { impl BackendExt for SimpleBackend {} +impl Default for SimpleBackend { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod tests { use std::collections::HashMap; diff --git a/crates/support/src/lib.rs b/crates/support/src/lib.rs index cd2cc3f..8391986 100644 --- a/crates/support/src/lib.rs +++ b/crates/support/src/lib.rs @@ -139,11 +139,9 @@ fn parse_file(content: &str, ext: &str, locale: &str) -> Result { - return Ok(parse_file_v1(locale, &v)); + Err("Invalid locale file format, please check the version field".into()) } + _ => Ok(parse_file_v1(locale, &v)), }, Err(e) => Err(e), } @@ -157,7 +155,7 @@ fn parse_file(content: &str, ext: &str, locale: &str) -> Result Translations { - return Translations::from([(locale.to_string(), data.clone())]); + Translations::from([(locale.to_string(), data.clone())]) } /// Locale file format v2 @@ -212,7 +210,7 @@ fn parse_file_v2(key_prefix: &str, data: &serde_json::Value) -> Option), iter them and convert them and insert into trs let key = format_keys(&[&key_prefix, &key]); - if let Some(sub_trs) = parse_file_v2(&key, &value) { + if let Some(sub_trs) = parse_file_v2(&key, value) { // println!("--------------- sub_trs:\n{:?}", sub_trs); // Merge the sub_trs into trs for (locale, sub_value) in sub_trs { @@ -241,7 +239,7 @@ fn get_version(data: &serde_json::Value) -> usize { return version.as_u64().unwrap_or(1) as usize; } - return 1; + 1 } /// Join the keys with dot, if any key is empty, omit it. From f8d816aceb2a708468291d637434ec301c08f8f2 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 15:31:44 +0800 Subject: [PATCH 32/39] Update README.md --- README.md | 131 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 89a9101..035b4d3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ > 🎯 Let's make I18n things to easy! -Rust I18n is a crate for loading localized text from a set of (YAML, JSON or TOML) mapping files. The mappings are converted into data readable by Rust programs at compile time, and then localized text can be loaded by simply calling the provided `t!` or `tr` macro. +Rust I18n is a crate for loading localized text from a set of (YAML, JSON or TOML) mapping files. The mappings are converted into data readable by Rust programs at compile time, and then localized text can be loaded by simply calling the provided [`t!`] macro. Unlike other I18n libraries, Rust I18n's goal is to provide a simple and easy-to-use API. @@ -13,10 +13,14 @@ The API of this crate is inspired by [ruby-i18n](https://github.com/ruby-i18n/i1 ## Features - Codegen on compile time for includes translations into binary. -- Global `t!` or `tr!` macro for loading localized text in everywhere. +- Global [`t!`] macro for loading localized text in everywhere. - Use YAML (default), JSON or TOML format for mapping localized text, and support mutiple files merging. - `cargo i18n` Command line tool for checking and extract untranslated texts into YAML files. - Support all localized texts in one file, or split into difference files by locale. +- Supports specifying a chain of fallback locales for missing translations. +- Supports automatic lookup of language territory for fallback locale. For instance, if `zh-CN` is not available, it will fallback to `zh`. (Since v2.4.0) +- Support short hashed keys for optimize memory usage and lookup speed. (Since v3.1.0) +- Support format variables in [`t!`], and support format variables with [`std::fmt`](https://doc.rust-lang.org/std/fmt/) syntax. (Since v3.1.0) ## Usage @@ -30,7 +34,7 @@ rust-i18n = "3" Load macro and init translations in `lib.rs` or `main.rs`: ```rust,compile_fail,no_run -// Load I18n macro, for allow you use `t!` or `tr!` macro in anywhere. +// Load I18n macro, for allow you use `t!` macro in anywhere. #[macro_use] extern crate rust_i18n; @@ -39,23 +43,39 @@ i18n!("locales"); // Or just use `i18n!`, default locales path is: "locales" in current crate. // -// i18n!(); +i18n!(); // Config fallback missing translations to "en" locale. // Use `fallback` option to set fallback locale. // -// i18n!("locales", fallback = "en"); +i18n!("locales", fallback = "en"); // Or more than one fallback with priority. // -// i18n!("locales", fallback = ["en", "es]); +i18n!("locales", fallback = ["en", "es"]); + +// Use a short hashed key as an identifier for long string literals +// to optimize memory usage and lookup speed. +// The key generation algorithm is `${Prefix}${Base62(SipHash13("msg"))}`. +i18n!("locales", minify_key = true); +// +// Alternatively, you can customize the key length, prefix, +// and threshold for the short hashed key. +i18n!("locales", + minify_key = true, + minify_key_len = 12, + minify_key_prefix = "T.", + minify_key_thresh = 64 +); +// Now, if the message length exceeds 64, the `t!` macro will automatically generate +// a 12-byte short hashed key with a "T." prefix for it, if not, it will use the original. ``` Or you can import by use directly: ```rust,no_run -// You must import in each files when you wants use `t!` or `tr!` macro. -use rust_i18n::{t, tr}; +// You must import in each files when you wants use `t!` macro. +use rust_i18n::t; rust_i18n::i18n!("locales"); @@ -63,9 +83,6 @@ fn main() { // Find the translation for the string literal `Hello` using the manually provided key `hello`. println!("{}", t!("hello")); - // Or, find the translation for the string literal `Hello` using the auto-generated key. - println!("{}", tr!("Hello")); - // Use `available_locales!` method to get all available locales. println!("{:?}", rust_i18n::available_locales!()); } @@ -97,8 +114,8 @@ _version: 1 hello: "Hello world" messages.hello: "Hello, %{name}" -# Key auto-generated by tr! -tr_4Cct6Q289b12SkvF47dXIx: "Hello, %{name}" +# Generate short hashed keys using `minify_key=true, minify_key_thresh=10` +4Cct6Q289b12SkvF47dXIx: "Hello, %{name}" ``` Or use JSON or TOML format, just rename the file to `en.json` or `en.toml`, and the content is like this: @@ -109,16 +126,16 @@ Or use JSON or TOML format, just rename the file to `en.json` or `en.toml`, and "hello": "Hello world", "messages.hello": "Hello, %{name}", - // Key auto-generated by tr! - "tr_4Cct6Q289b12SkvF47dXIx": "Hello, %{name}" + // Generate short hashed keys using `minify_key=true, minify_key_thresh=10` + "4Cct6Q289b12SkvF47dXIx": "Hello, %{name}" } ``` ```toml hello = "Hello world" -# Key auto-generated by tr! -tr_4Cct6Q289b12SkvF47dXIx = "Hello, %{name}" +# Generate short hashed keys using `minify_key=true, minify_key_thresh=10` +4Cct6Q289b12SkvF47dXIx = "Hello, %{name}" [messages] hello = "Hello, %{name}" @@ -158,8 +175,8 @@ messages.hello: en: Hello, %{name} zh-CN: 你好,%{name} -# Key auto-generated by tr! -tr_4Cct6Q289b12SkvF47dXIx: +# Generate short hashed keys using `minify_key=true, minify_key_thresh=10` +4Cct6Q289b12SkvF47dXIx: en: Hello, %{name} zh-CN: 你好,%{name} ``` @@ -170,20 +187,20 @@ This is useful when you use [GitHub Copilot](https://github.com/features/copilot ### Get Localized Strings in Rust -Import the `t!` or `tr` macro from this crate into your current scope: +Import the [`t!`] macro from this crate into your current scope: ```rust,no_run -use rust_i18n::{t, tr}; +use rust_i18n::t; ``` Then, simply use it wherever a localized string is needed: ```rust,no_run -# use rust_i18n::CowStr; -# fn _rust_i18n_translate(locale: &str, key: &str) -> String { todo!() } -# fn _rust_i18n_try_translate<'r>(locale: &str, key: &'r str) -> Option> { todo!() } +# macro_rules! t { +# ($($all_tokens:tt)*) => {} +# } # fn main() { -use rust_i18n::{t, tr}; +// use rust_i18n::t; t!("hello"); // => "Hello world" @@ -202,31 +219,14 @@ t!("messages.hello", locale = "zh-CN", name = "Jason", count = 2); t!("messages.hello", locale = "zh-CN", "name" => "Jason", "count" => 3 + 2); // => "你好,Jason (5)" -tr!("Hello world"); -// => "Hello world" (Key `tr_3RnEdpgZvZ2WscJuSlQJkJ` for "Hello world") - -tr!("Hello world", locale = "de"); -// => "Hallo Welt!" (Key `tr_3RnEdpgZvZ2WscJuSlQJkJ` for "Hello world") - -tr!("Hello, %{name}", name = "world"); -// => "Hello, world" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hello, %{name}") - -tr!("Hello, %{name} and %{other}", name = "Foo", other = "Bar"); -// => "Hello, Foo and Bar" (Key `tr_3eULVGYoyiBuaM27F93Mo7` for "Hello, %{name} and %{other}") - -tr!("Hello, %{name}", locale = "de", name = "Jason"); -// => "Hallo, Jason" (Key `tr_4Cct6Q289b12SkvF47dXIx` for "Hallo, %{name}") - -tr!("Hello, %{name}, you serial number is: %{sn}", name = "Jason", sn = 123 : {:08}); +t!("Hello, %{name}, you serial number is: %{sn}", name = "Jason", sn = 123 : {:08}); // => "Hello, Jason, you serial number is: 000000123" # } ``` -💡 NOTE: The key `tr_3RnEdpgZvZ2WscJuSlQJkJ` is auto-generated by `tr!()`, and can be extract from source code to localized files with `rust-i18n-cli`. - ### Current Locale -You can use `rust_i18n::set_locale` to set the global locale at runtime, so that you don't have to specify the locale on each `t!` or `tr` invocation. +You can use [`rust_i18n::set_locale()`]() to set the global locale at runtime, so that you don't have to specify the locale on each [`t!`] invocation. ```rust rust_i18n::set_locale("zh-CN"); @@ -299,7 +299,7 @@ rust_i18n::i18n!("locales", backend = RemoteI18n::new()); This also will load local translates from ./locales path, but your own `RemoteI18n` will priority than it. -Now you call `t!` or `tr!` will lookup translates from your own backend first, if not found, will lookup from local files. +Now you call [`t!`] will lookup translates from your own backend first, if not found, will lookup from local files. ## Example @@ -309,7 +309,7 @@ A minimal example of using rust-i18n can be found [here](https://github.com/long I18n Ally is a VS Code extension for helping you translate your Rust project. -You can add [i18n-ally-custom-framework.yml](https://github.com/longbridgeapp/rust-i18n/blob/main/.vscode/i18n-ally-custom-framework.yml) to your project `.vscode` directory, and then use I18n Ally can parse `t!` or `tr!` marco to show translate text in VS Code editor. +You can add [i18n-ally-custom-framework.yml](https://github.com/longbridgeapp/rust-i18n/blob/main/.vscode/i18n-ally-custom-framework.yml) to your project `.vscode` directory, and then use I18n Ally can parse `t!` marco to show translate text in VS Code editor. ## Extractor @@ -381,24 +381,39 @@ Run `cargo i18n -h` to see details. ```bash $ cargo i18n -h -cargo-i18n 0.5.0 +cargo-i18n 3.1.0 --------------------------------------- -Rust I18n command for help you simply to extract all untranslated texts from soruce code. +Rust I18n command to help you extract all untranslated texts from source code. -It will iter all Rust files in and extract all untranslated texts that used `t!` macro. -And then generate a YAML file and merge for existing texts. +It will iterate all Rust files in the source directory and extract all untranslated texts that used `t!` macro. Then it will generate a YAML file and merge with the existing translations. https://github.com/longbridgeapp/rust-i18n -USAGE: - cargo i18n [OPTIONS] [--] [source] +Usage: cargo i18n [OPTIONS] [-- ] + +Arguments: + [SOURCE] + Extract all untranslated I18n texts from source code + + [default: ./] + +Options: + -t, --translate ... + Manually add a translation to the localization file. + + This is useful for non-literal values in the `t!` macro. + + For example, if you have `t!(format!("Hello, {}!", "world"))` in your code, + you can add a translation for it using `-t "Hello, world!"`, + or provide a translated message using `-t "Hello, world! => Hola, world!"`. + + NOTE: The whitespace before and after the key and value will be trimmed. -FLAGS: - -h, --help Prints help information - -V, --version Prints version information + -h, --help + Print help (see a summary with '-h') -ARGS: - Path of your Rust crate root [default: ./] + -V, --version + Print version ``` ## Debugging the Codegen Process @@ -411,7 +426,7 @@ $ RUST_I18N_DEBUG=1 cargo build ## Benchmark -Benchmark `t!` method, result on Apple M1: +Benchmark [`t!`] method, result on Apple M1: ```bash t time: [58.274 ns 60.222 ns 62.390 ns] From 8b325d3d1448445c64c2933789cfa8a72559258a Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 16:36:48 +0800 Subject: [PATCH 33/39] Rename feature `log-missing` to `log-miss-tr` --- Cargo.toml | 6 ++-- README.md | 1 + crates/macro/Cargo.toml | 2 +- crates/macro/src/tr.rs | 20 ++++++++----- examples/app-egui/Cargo.toml | 3 +- examples/app-egui/src/main.rs | 46 ++++++++++++++++++++++++----- examples/app-minify-key/Cargo.toml | 2 +- examples/app-minify-key/src/main.rs | 6 ++-- examples/atomic_str/Cargo.toml | 9 ++++++ tests/i18n_minify_key.rs | 1 - 10 files changed, 71 insertions(+), 25 deletions(-) create mode 100644 examples/atomic_str/Cargo.toml diff --git a/Cargo.toml b/Cargo.toml index 0303467..d0942e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,9 @@ serde_yaml = "0.8" globwalk = "0.8.1" regex = "1" +[features] +log-miss-tr = ["rust-i18n-macro/log-miss-tr"] + [[example]] name = "app" test = true @@ -51,6 +54,3 @@ name = "bench" [[bench]] harness = false name = "minify_key" - -[features] -log-missing = ["rust-i18n-macro/log-missing"] diff --git a/README.md b/README.md index 035b4d3..06ae501 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The API of this crate is inspired by [ruby-i18n](https://github.com/ruby-i18n/i1 - Supports automatic lookup of language territory for fallback locale. For instance, if `zh-CN` is not available, it will fallback to `zh`. (Since v2.4.0) - Support short hashed keys for optimize memory usage and lookup speed. (Since v3.1.0) - Support format variables in [`t!`], and support format variables with [`std::fmt`](https://doc.rust-lang.org/std/fmt/) syntax. (Since v3.1.0) +- Support for log missing translations at the warning level with `log-miss-tr` feature, the feature requires the `log` crate. (Since v3.1.0) ## Usage diff --git a/crates/macro/Cargo.toml b/crates/macro/Cargo.toml index fe9dfad..2661b3e 100644 --- a/crates/macro/Cargo.toml +++ b/crates/macro/Cargo.toml @@ -27,4 +27,4 @@ rust-i18n = { path = "../.." } proc-macro = true [features] -log-missing = [] +log-miss-tr = [] diff --git a/crates/macro/src/tr.rs b/crates/macro/src/tr.rs index 1ba9bef..77b479a 100644 --- a/crates/macro/src/tr.rs +++ b/crates/macro/src/tr.rs @@ -374,6 +374,18 @@ impl Tr { Ok(()) } + #[cfg(feature = "log-miss-tr")] + fn log_missing() -> proc_macro2::TokenStream { + quote! { + log::log!(target: "rust-i18n", log::Level::Warn, "missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); + } + } + + #[cfg(not(feature = "log-miss-tr"))] + fn log_missing() -> proc_macro2::TokenStream { + quote! {} + } + fn into_token_stream(self) -> proc_macro2::TokenStream { let (msg_key, msg_val) = if self.minify_key && self.msg.val.is_expr_lit_str() { let msg_val = self.msg.val.to_string().unwrap(); @@ -416,13 +428,7 @@ impl Tr { quote! { format!(#sepecifiers, #value) } }) .collect(); - let logging = if cfg!(feature = "log-missing") { - quote! { - log::log!(target: "rust-i18n", log::Level::Warn, "missing: {} => {:?} @ {}:{}", msg_key, msg_val, file!(), line!()); - } - } else { - quote! {} - }; + let logging = Self::log_missing(); if self.args.is_empty() { quote! { { diff --git a/examples/app-egui/Cargo.toml b/examples/app-egui/Cargo.toml index a046a7a..97a1a73 100644 --- a/examples/app-egui/Cargo.toml +++ b/examples/app-egui/Cargo.toml @@ -12,8 +12,7 @@ log = { version = "0.4", optional = true } rust-i18n = { path = "../.." } [features] -default = ["log-missing"] -log-missing = ["env_logger", "log", "rust-i18n/log-missing"] +log-miss-tr = ["env_logger", "log", "rust-i18n/log-miss-tr"] [package.metadata.i18n] available-locales = ["en", "fr", "ja", "ko", "zh", "zh-CN"] diff --git a/examples/app-egui/src/main.rs b/examples/app-egui/src/main.rs index 4462942..3c5b9a9 100644 --- a/examples/app-egui/src/main.rs +++ b/examples/app-egui/src/main.rs @@ -12,6 +12,7 @@ rust_i18n::i18n!( ); fn main() -> Result<(), eframe::Error> { + #[cfg(feature = "log-miss-tr")] env_logger::init(); // Log to stderr (if you run with `RUST_LOG=debug`). let options = eframe::NativeOptions { viewport: egui::ViewportBuilder::default().with_inner_size([320.0, 240.0]), @@ -23,7 +24,7 @@ fn main() -> Result<(), eframe::Error> { Box::new(|cc| { // This gives us image support: // egui_extras::install_image_loaders(&cc.egui_ctx); - setup_custom_fonts(&cc.egui_ctx); + let _ = setup_custom_fonts(&cc.egui_ctx); Box::::default() }), ) @@ -80,17 +81,38 @@ impl eframe::App for MyApp { } } -fn setup_custom_fonts(ctx: &egui::Context) { +#[cfg(windows)] +fn try_load_system_font() -> Result, std::io::Error> { + let font_files = &[ + "C:/Windows/Fonts/msyh.ttc", + "C:/Windows/Fonts/msjh.ttf", + "C:/Windows/Fonts/yugothr.ttc", + "C:/Windows/Fonts/malgun.ttf", + ]; + + for font in font_files { + if let Ok(font) = std::fs::read(font) { + return Ok(font); + } + } + + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "No system font found", + )) +} + +#[cfg(windows)] +fn setup_custom_fonts(ctx: &egui::Context) -> Result<(), std::io::Error> { // Start with the default fonts (we will be adding to them rather than replacing them). let mut fonts = egui::FontDefinitions::default(); // Install my own font (maybe supporting non-latin characters). // .ttf and .otf files supported. - // NOTE: only support Windows and Simplified Chinese for now. - fonts.font_data.insert( - "my_font".to_owned(), - egui::FontData::from_static(include_bytes!("C:/Windows/Fonts/msyh.ttc")), - ); + let font_bytes = try_load_system_font()?; + fonts + .font_data + .insert("my_font".to_owned(), egui::FontData::from_owned(font_bytes)); // Put my font first (highest priority) for proportional text: fonts @@ -108,4 +130,14 @@ fn setup_custom_fonts(ctx: &egui::Context) { // Tell egui to use these fonts: ctx.set_fonts(fonts); + + Ok(()) +} + +#[cfg(not(windows))] +fn setup_custom_fonts(ctx: &egui::Context) -> Result<(), std::io::Error> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "Custom fonts not supported on this platform", + )) } diff --git a/examples/app-minify-key/Cargo.toml b/examples/app-minify-key/Cargo.toml index 952e9c2..4ef9c15 100644 --- a/examples/app-minify-key/Cargo.toml +++ b/examples/app-minify-key/Cargo.toml @@ -11,7 +11,7 @@ log = { version = "0.4", optional = true } rust-i18n = { path = "../.." } [features] -log-missing = ["env_logger", "log", "rust-i18n/log-missing"] +log-miss-tr = ["env_logger", "log", "rust-i18n/log-miss-tr"] [package.metadata.i18n] available-locales = ["en", "zh-CN"] diff --git a/examples/app-minify-key/src/main.rs b/examples/app-minify-key/src/main.rs index 2091519..2b714e8 100644 --- a/examples/app-minify-key/src/main.rs +++ b/examples/app-minify-key/src/main.rs @@ -8,7 +8,7 @@ rust_i18n::i18n!( minify_key_thresh = 4 ); -#[cfg(feature = "log-missing")] +#[cfg(feature = "log-miss-tr")] fn set_logger() { env_logger::builder() .filter_level(log::LevelFilter::Debug) @@ -16,7 +16,7 @@ fn set_logger() { .init(); } -#[cfg(not(feature = "log-missing"))] +#[cfg(not(feature = "log-miss-tr"))] fn set_logger() {} fn main() { @@ -76,7 +76,7 @@ fn main() { } println!(); - if cfg!(feature = "log-missing") { + if cfg!(feature = "log-miss-tr") { println!("Translates runtime strings and logs when a lookup is missing:"); for locale in &locales { let msg = "Foo Bar".to_string(); diff --git a/examples/atomic_str/Cargo.toml b/examples/atomic_str/Cargo.toml new file mode 100644 index 0000000..694fd19 --- /dev/null +++ b/examples/atomic_str/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "atomic_str" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rust-i18n = { path = "../../" } diff --git a/tests/i18n_minify_key.rs b/tests/i18n_minify_key.rs index f25d6cc..27c8445 100644 --- a/tests/i18n_minify_key.rs +++ b/tests/i18n_minify_key.rs @@ -9,7 +9,6 @@ rust_i18n::i18n!( #[cfg(test)] mod tests { - use super::*; use rust_i18n::{t, tkv}; #[test] From a11224fe9061d0dca0212cf17040db041dcd917c Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 19:20:05 +0800 Subject: [PATCH 34/39] Cargo.toml: apply toml format --- Cargo.toml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d0942e7..e4dcd32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,17 @@ categories = ["localization", "internationalization"] description = "Rust I18n is use Rust codegen for load YAML file storage translations on compile time, and give you a t! macro for simply get translation texts." edition = "2021" exclude = ["crates", "tests"] -keywords = ["gettext", "i18n", "l10n", "intl", "internationalization", "localization", "tr", "translation", "yml"] +keywords = [ + "gettext", + "i18n", + "l10n", + "intl", + "internationalization", + "localization", + "tr", + "translation", + "yml", +] license = "MIT" name = "rust-i18n" readme = "README.md" From 0b606cc27e172ea9412ad8fc979bd87317336843 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 22:51:14 +0800 Subject: [PATCH 35/39] Move `I18nConfig` from `rust-i18n-cli` to `rust-i18n-support` --- crates/cli/Cargo.toml | 3 - crates/cli/src/main.rs | 24 ++--- crates/extract/Cargo.toml | 2 +- crates/extract/src/attrs.rs | 47 -------- crates/extract/src/extractor.rs | 21 ++-- crates/extract/src/lib.rs | 1 - crates/support/Cargo.toml | 3 +- crates/{cli => support}/src/config.rs | 148 +++++++++++++++----------- crates/support/src/lib.rs | 2 + 9 files changed, 115 insertions(+), 136 deletions(-) delete mode 100644 crates/extract/src/attrs.rs rename crates/{cli => support}/src/config.rs (52%) diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 99a1440..d397d72 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -10,11 +10,8 @@ version = "3.0.0" [dependencies] anyhow = "1" clap = { version = "4.1.14", features = ["derive"] } -itertools = "0.11.0" rust-i18n-support = { path = "../support", version = "3.0.0" } rust-i18n-extract = { path = "../extract", version = "3.0.0" } -serde = { version = "1", features = ["derive"] } -toml = "0.7.4" [[bin]] name = "cargo-i18n" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 1d75711..0a5dc13 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,13 +1,9 @@ use anyhow::Error; use clap::{Args, Parser}; -use rust_i18n_extract::attrs::I18nAttrs; -use rust_i18n_support::MinifyKey; - -use std::{collections::HashMap, path::Path}; - use rust_i18n_extract::extractor::Message; use rust_i18n_extract::{extractor, generator, iter}; -mod config; +use rust_i18n_support::{I18nConfig, MinifyKey}; +use std::{collections::HashMap, path::Path}; #[derive(Parser)] #[command(name = "cargo")] @@ -71,14 +67,18 @@ fn translate_value_parser(s: &str) -> Result<(String, String), std::io::Error> { fn add_translations( list: &[(String, String)], results: &mut HashMap, - attrs: &I18nAttrs, + cfg: &I18nConfig, ) { - let I18nAttrs { + let I18nConfig { + default_locale: _, + available_locales: _, + load_path: _, minify_key, minify_key_len, minify_key_prefix, minify_key_thresh, - } = attrs; + } = cfg; + for item in list { let index = results.len(); let key = if *minify_key { @@ -105,14 +105,14 @@ fn main() -> Result<(), Error> { let source_path = args.source.expect("Missing source path"); - let cfg = config::load(std::path::Path::new(&source_path))?; + let cfg = I18nConfig::load(std::path::Path::new(&source_path))?; iter::iter_crate(&source_path, |path, source| { - extractor::extract(&mut results, path, source, cfg.attrs.clone()) + extractor::extract(&mut results, path, source, cfg.clone()) })?; if let Some(list) = args.translate { - add_translations(&list, &mut results, &cfg.attrs); + add_translations(&list, &mut results, &cfg); } let mut messages: Vec<_> = results.iter().collect(); diff --git a/crates/extract/Cargo.toml b/crates/extract/Cargo.toml index fd83d8f..f4522cd 100644 --- a/crates/extract/Cargo.toml +++ b/crates/extract/Cargo.toml @@ -16,7 +16,7 @@ proc-macro2 = { version = "1", features = ["span-locations"] } quote = "1" regex = "1" rust-i18n-support = { path = "../support", version = "3.0.0" } -serde = { version = "1", features = ["derive"] } +serde = "1" serde_json = "1" serde_yaml = "0.8" syn = { version = "2.0.18", features = ["full"] } diff --git a/crates/extract/src/attrs.rs b/crates/extract/src/attrs.rs deleted file mode 100644 index eb8c954..0000000 --- a/crates/extract/src/attrs.rs +++ /dev/null @@ -1,47 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -pub struct I18nAttrs { - #[serde(default = "minify_key")] - pub minify_key: bool, - #[serde(default = "minify_key_len")] - pub minify_key_len: usize, - #[serde(default = "minify_key_prefix")] - pub minify_key_prefix: String, - #[serde(default = "minify_key_thresh")] - pub minify_key_thresh: usize, -} - -impl I18nAttrs { - pub fn new() -> Self { - Self { - minify_key: rust_i18n_support::DEFAULT_MINIFY_KEY, - minify_key_len: rust_i18n_support::DEFAULT_MINIFY_KEY_LEN, - minify_key_prefix: rust_i18n_support::DEFAULT_MINIFY_KEY_PREFIX.to_string(), - minify_key_thresh: rust_i18n_support::DEFAULT_MINIFY_KEY_THRESH, - } - } -} - -impl Default for I18nAttrs { - fn default() -> Self { - Self::new() - } -} - -fn minify_key() -> bool { - I18nAttrs::default().minify_key -} - -fn minify_key_len() -> usize { - I18nAttrs::default().minify_key_len -} - -fn minify_key_prefix() -> String { - I18nAttrs::default().minify_key_prefix -} - -fn minify_key_thresh() -> usize { - I18nAttrs::default().minify_key_thresh -} diff --git a/crates/extract/src/extractor.rs b/crates/extract/src/extractor.rs index 20e40d4..a2bb675 100644 --- a/crates/extract/src/extractor.rs +++ b/crates/extract/src/extractor.rs @@ -1,7 +1,7 @@ -use crate::attrs::I18nAttrs; use anyhow::Error; use proc_macro2::{TokenStream, TokenTree}; use quote::ToTokens; +use rust_i18n_support::I18nConfig; use std::collections::HashMap; use std::path::PathBuf; @@ -39,13 +39,9 @@ pub fn extract( results: &mut Results, path: &PathBuf, source: &str, - attrs: I18nAttrs, + cfg: I18nConfig, ) -> Result<(), Error> { - let mut ex = Extractor { - results, - path, - attrs, - }; + let mut ex = Extractor { results, path, cfg }; let file = syn::parse_file(source) .unwrap_or_else(|_| panic!("Failed to parse file, file: {}", path.display())); @@ -57,7 +53,7 @@ pub fn extract( struct Extractor<'a> { results: &'a mut Results, path: &'a PathBuf, - attrs: I18nAttrs, + cfg: I18nConfig, } impl<'a> Extractor<'a> { @@ -99,12 +95,15 @@ impl<'a> Extractor<'a> { return; }; - let I18nAttrs { + let I18nConfig { + default_locale: _, + available_locales: _, + load_path: _, minify_key, minify_key_len, minify_key_prefix, minify_key_thresh, - } = &self.attrs; + } = &self.cfg; let key: Option = Some(literal); if let Some(lit) = key { @@ -242,7 +241,7 @@ mod tests { let mut ex = Extractor { results: &mut results, path: &"hello.rs".to_owned().into(), - attrs: I18nAttrs::default(), + cfg: I18nConfig::default(), }; ex.invoke(stream).unwrap(); diff --git a/crates/extract/src/lib.rs b/crates/extract/src/lib.rs index f55cb26..9d03299 100644 --- a/crates/extract/src/lib.rs +++ b/crates/extract/src/lib.rs @@ -1,4 +1,3 @@ -pub mod attrs; pub mod extractor; pub mod generator; pub mod iter; diff --git a/crates/support/Cargo.toml b/crates/support/Cargo.toml index d59ac21..a556254 100644 --- a/crates/support/Cargo.toml +++ b/crates/support/Cargo.toml @@ -11,9 +11,10 @@ version = "3.0.1" arc-swap = "1.6.0" base62 = "2.0.2" globwalk = "0.8.1" +itertools = "0.11.0" once_cell = "1.10.0" proc-macro2 = "1.0" -serde = "1" +serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.8" siphasher = "1.0" diff --git a/crates/cli/src/config.rs b/crates/support/src/config.rs similarity index 52% rename from crates/cli/src/config.rs rename to crates/support/src/config.rs index 888d9e6..e66d76b 100644 --- a/crates/cli/src/config.rs +++ b/crates/support/src/config.rs @@ -4,14 +4,13 @@ //! See `Manifest::from_slice`. use itertools::Itertools; -use rust_i18n_extract::attrs::I18nAttrs; use serde::{Deserialize, Serialize}; use std::fs; use std::io; use std::io::Read; use std::path::Path; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub struct I18nConfig { #[serde(default = "default_locale")] @@ -20,8 +19,66 @@ pub struct I18nConfig { pub available_locales: Vec, #[serde(default = "load_path")] pub load_path: String, - #[serde(flatten)] - pub attrs: I18nAttrs, + #[serde(default = "minify_key")] + pub minify_key: bool, + #[serde(default = "minify_key_len")] + pub minify_key_len: usize, + #[serde(default = "minify_key_prefix")] + pub minify_key_prefix: String, + #[serde(default = "minify_key_thresh")] + pub minify_key_thresh: usize, +} + +impl I18nConfig { + pub fn new() -> Self { + Self { + default_locale: "en".to_string(), + available_locales: vec!["en".to_string()], + load_path: "./locales".to_string(), + minify_key: crate::DEFAULT_MINIFY_KEY, + minify_key_len: crate::DEFAULT_MINIFY_KEY_LEN, + minify_key_prefix: crate::DEFAULT_MINIFY_KEY_PREFIX.to_string(), + minify_key_thresh: crate::DEFAULT_MINIFY_KEY_THRESH, + } + } + + pub fn load(cargo_root: &Path) -> io::Result { + let cargo_file = cargo_root.join("Cargo.toml"); + let mut file = fs::File::open(&cargo_file) + .unwrap_or_else(|e| panic!("Fail to open {}, {}", cargo_file.display(), e)); + + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + Self::parse(&contents) + } + + pub fn parse(contents: &str) -> io::Result { + if !contents.contains("[i18n]") && !contents.contains("[package.metadata.i18n]") { + return Ok(I18nConfig::default()); + } + let contents = contents.replace("[package.metadata.i18n]", "[i18n]"); + let mut config: MainConfig = toml::from_str(&contents) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; + + // Push default_locale + config + .i18n + .available_locales + .insert(0, config.i18n.default_locale.clone()); + + // unqiue + config.i18n.available_locales = + config.i18n.available_locales.into_iter().unique().collect(); + + Ok(config.i18n) + } +} + +impl Default for I18nConfig { + fn default() -> Self { + Self::new() + } } fn default_locale() -> String { @@ -36,52 +93,26 @@ fn load_path() -> String { I18nConfig::default().load_path } -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -#[serde(rename_all = "kebab-case")] -pub struct MainConfig { - pub i18n: I18nConfig, +fn minify_key() -> bool { + I18nConfig::default().minify_key } -impl Default for I18nConfig { - fn default() -> Self { - I18nConfig { - default_locale: "en".to_string(), - available_locales: vec!["en".to_string()], - load_path: "./locales".to_string(), - attrs: I18nAttrs::default(), - } - } +fn minify_key_len() -> usize { + I18nConfig::default().minify_key_len } -pub fn load(cargo_root: &Path) -> io::Result { - let cargo_file = cargo_root.join("Cargo.toml"); - let mut file = fs::File::open(&cargo_file) - .unwrap_or_else(|e| panic!("Fail to open {}, {}", cargo_file.display(), e)); - - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - - parse(&contents) +fn minify_key_prefix() -> String { + I18nConfig::default().minify_key_prefix } -pub fn parse(contents: &str) -> io::Result { - if !contents.contains("[i18n]") && !contents.contains("[package.metadata.i18n]") { - return Ok(I18nConfig::default()); - } - let contents = contents.replace("[package.metadata.i18n]", "[i18n]"); - let mut config: MainConfig = toml::from_str(&contents) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e.to_string()))?; - - // Push default_locale - config - .i18n - .available_locales - .insert(0, config.i18n.default_locale.clone()); - - // unqiue - config.i18n.available_locales = config.i18n.available_locales.into_iter().unique().collect(); +fn minify_key_thresh() -> usize { + I18nConfig::default().minify_key_thresh +} - Ok(config.i18n) +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub struct MainConfig { + pub i18n: I18nConfig, } #[test] @@ -97,32 +128,30 @@ fn test_parse() { minify-key-thresh = 16 "#; - let cfg = parse(contents).unwrap(); + let cfg = I18nConfig::parse(contents).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); assert_eq!(cfg.load_path, "./my-locales"); - assert_eq!(cfg.attrs.minify_key, true); - assert_eq!(cfg.attrs.minify_key_len, 12); - assert_eq!(cfg.attrs.minify_key_prefix, "T."); - assert_eq!(cfg.attrs.minify_key_thresh, 16); + assert_eq!(cfg.minify_key, true); + assert_eq!(cfg.minify_key_len, 12); + assert_eq!(cfg.minify_key_prefix, "T."); + assert_eq!(cfg.minify_key_thresh, 16); let contents = r#" [i18n] available-locales = ["zh-CN", "de", "de"] load-path = "./my-locales" "#; - let cfg = parse(contents).unwrap(); + let cfg = I18nConfig::parse(contents).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN", "de"]); assert_eq!(cfg.load_path, "./my-locales"); - assert_eq!(cfg.attrs, I18nAttrs::default()); let contents = ""; - let cfg = parse(contents).unwrap(); + let cfg = I18nConfig::parse(contents).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en"]); assert_eq!(cfg.load_path, "./locales"); - assert_eq!(cfg.attrs, I18nAttrs::default()); } #[test] @@ -138,25 +167,24 @@ fn test_parse_with_metadata() { minify-key-thresh = 16 "#; - let cfg = parse(contents).unwrap(); + let cfg = I18nConfig::parse(contents).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); assert_eq!(cfg.load_path, "./my-locales"); - assert_eq!(cfg.attrs.minify_key, true); - assert_eq!(cfg.attrs.minify_key_len, 12); - assert_eq!(cfg.attrs.minify_key_prefix, "T."); - assert_eq!(cfg.attrs.minify_key_thresh, 16); + assert_eq!(cfg.minify_key, true); + assert_eq!(cfg.minify_key_len, 12); + assert_eq!(cfg.minify_key_prefix, "T."); + assert_eq!(cfg.minify_key_thresh, 16); } #[test] fn test_load_default() { let workdir = Path::new(env!["CARGO_MANIFEST_DIR"]); - let cfg = load(workdir).unwrap(); + let cfg = I18nConfig::load(workdir).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en"]); assert_eq!(cfg.load_path, "./locales"); - assert_eq!(cfg.attrs, I18nAttrs::default()); } #[test] @@ -164,7 +192,7 @@ fn test_load() { let workdir = Path::new(env!["CARGO_MANIFEST_DIR"]); let cargo_root = workdir.join("../../examples/foo"); - let cfg = load(&cargo_root).unwrap(); + let cfg = I18nConfig::load(&cargo_root).unwrap(); assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); } diff --git a/crates/support/src/lib.rs b/crates/support/src/lib.rs index 8391986..f721c5a 100644 --- a/crates/support/src/lib.rs +++ b/crates/support/src/lib.rs @@ -5,10 +5,12 @@ use std::{collections::HashMap, path::Path}; mod atomic_str; mod backend; +mod config; mod cow_str; mod minify_key; pub use atomic_str::AtomicStr; pub use backend::{Backend, BackendExt, SimpleBackend}; +pub use config::I18nConfig; pub use cow_str::CowStr; pub use minify_key::{ minify_key, MinifyKey, DEFAULT_MINIFY_KEY, DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, From f33d24d442a5db365620a2d05519d228504b5531 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Sun, 28 Jan 2024 23:26:59 +0800 Subject: [PATCH 36/39] Display the file path when loading a locale file fails --- crates/support/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/support/src/lib.rs b/crates/support/src/lib.rs index f721c5a..9dafd9d 100644 --- a/crates/support/src/lib.rs +++ b/crates/support/src/lib.rs @@ -105,7 +105,8 @@ pub fn load_locales bool>( .read_to_string(&mut content) .expect("Read file failed."); - let trs = parse_file(&content, ext, locale).expect("Parse file failed."); + let trs = parse_file(&content, ext, locale) + .expect(&format!("Parse file `{}` failed", entry.display())); trs.into_iter().for_each(|(k, new_value)| { translations From 2a9e8cf401ae39b640283876229a3e5fbae64185 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Mon, 29 Jan 2024 00:10:05 +0800 Subject: [PATCH 37/39] Add `fallback` option to I18nConfig --- crates/cli/src/main.rs | 4 +--- crates/extract/src/extractor.rs | 4 +--- crates/support/src/config.rs | 11 +++++++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 0a5dc13..d33932e 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -70,13 +70,11 @@ fn add_translations( cfg: &I18nConfig, ) { let I18nConfig { - default_locale: _, - available_locales: _, - load_path: _, minify_key, minify_key_len, minify_key_prefix, minify_key_thresh, + .. } = cfg; for item in list { diff --git a/crates/extract/src/extractor.rs b/crates/extract/src/extractor.rs index a2bb675..a64be79 100644 --- a/crates/extract/src/extractor.rs +++ b/crates/extract/src/extractor.rs @@ -96,13 +96,11 @@ impl<'a> Extractor<'a> { }; let I18nConfig { - default_locale: _, - available_locales: _, - load_path: _, minify_key, minify_key_len, minify_key_prefix, minify_key_thresh, + .. } = &self.cfg; let key: Option = Some(literal); diff --git a/crates/support/src/config.rs b/crates/support/src/config.rs index e66d76b..7917b4e 100644 --- a/crates/support/src/config.rs +++ b/crates/support/src/config.rs @@ -19,6 +19,8 @@ pub struct I18nConfig { pub available_locales: Vec, #[serde(default = "load_path")] pub load_path: String, + #[serde(default = "fallback")] + pub fallback: Vec, #[serde(default = "minify_key")] pub minify_key: bool, #[serde(default = "minify_key_len")] @@ -35,6 +37,7 @@ impl I18nConfig { default_locale: "en".to_string(), available_locales: vec!["en".to_string()], load_path: "./locales".to_string(), + fallback: vec![], minify_key: crate::DEFAULT_MINIFY_KEY, minify_key_len: crate::DEFAULT_MINIFY_KEY_LEN, minify_key_prefix: crate::DEFAULT_MINIFY_KEY_PREFIX.to_string(), @@ -93,6 +96,10 @@ fn load_path() -> String { I18nConfig::default().load_path } +fn fallback() -> Vec { + I18nConfig::default().fallback +} + fn minify_key() -> bool { I18nConfig::default().minify_key } @@ -122,6 +129,7 @@ fn test_parse() { default-locale = "en" available-locales = ["zh-CN"] load-path = "./my-locales" + fallback = ["zh"] minify-key = true minify-key-len = 12 minify-key-prefix = "T." @@ -132,6 +140,7 @@ fn test_parse() { assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); assert_eq!(cfg.load_path, "./my-locales"); + assert_eq!(cfg.fallback, vec!["zh"]); assert_eq!(cfg.minify_key, true); assert_eq!(cfg.minify_key_len, 12); assert_eq!(cfg.minify_key_prefix, "T."); @@ -161,6 +170,7 @@ fn test_parse_with_metadata() { default-locale = "en" available-locales = ["zh-CN"] load-path = "./my-locales" + fallback = ["zh"] minify-key = true minify-key-len = 12 minify-key-prefix = "T." @@ -171,6 +181,7 @@ fn test_parse_with_metadata() { assert_eq!(cfg.default_locale, "en"); assert_eq!(cfg.available_locales, vec!["en", "zh-CN"]); assert_eq!(cfg.load_path, "./my-locales"); + assert_eq!(cfg.fallback, vec!["zh"]); assert_eq!(cfg.minify_key, true); assert_eq!(cfg.minify_key_len, 12); assert_eq!(cfg.minify_key_prefix, "T."); From 8762c15609f33b959b6d5365d7438f9ecafea97a Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Mon, 29 Jan 2024 00:23:10 +0800 Subject: [PATCH 38/39] Add `metadata` supports to `in18n!` --- Cargo.toml | 1 + README.md | 4 ++ crates/macro/src/lib.rs | 59 +++++++++++++++++++++++++++- examples/app-metadata/Cargo.toml | 18 +++++++++ examples/app-metadata/locales/v2.yml | 23 +++++++++++ examples/app-metadata/src/main.rs | 24 +++++++++++ tests/integration_tests.rs | 4 ++ 7 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 examples/app-metadata/Cargo.toml create mode 100644 examples/app-metadata/locales/v2.yml create mode 100644 examples/app-metadata/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index e4dcd32..da04644 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ members = [ "crates/macro", "examples/app-egui", "examples/app-load-path", + "examples/app-metadata", "examples/app-minify-key", "examples/foo", ] diff --git a/README.md b/README.md index 06ae501..2af2c22 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,10 @@ i18n!("locales", ); // Now, if the message length exceeds 64, the `t!` macro will automatically generate // a 12-byte short hashed key with a "T." prefix for it, if not, it will use the original. + +// Configuration using the `[package.metadata.i18n]` section in `Cargo.toml`, +// Useful for the `cargo i18n` command line tool. +i18n!(metadata = true); ``` Or you can import by use directly: diff --git a/crates/macro/src/lib.rs b/crates/macro/src/lib.rs index aba5842..78beb85 100644 --- a/crates/macro/src/lib.rs +++ b/crates/macro/src/lib.rs @@ -1,7 +1,7 @@ use quote::quote; use rust_i18n_support::{ - is_debug, load_locales, DEFAULT_MINIFY_KEY, DEFAULT_MINIFY_KEY_LEN, DEFAULT_MINIFY_KEY_PREFIX, - DEFAULT_MINIFY_KEY_THRESH, + is_debug, load_locales, I18nConfig, DEFAULT_MINIFY_KEY, DEFAULT_MINIFY_KEY_LEN, + DEFAULT_MINIFY_KEY_PREFIX, DEFAULT_MINIFY_KEY_THRESH, }; use std::collections::HashMap; use syn::{parse_macro_input, Expr, Ident, LitBool, LitStr, Token}; @@ -11,8 +11,10 @@ mod tr; struct Args { locales_path: String, + default_locale: Option, fallback: Option>, extend: Option, + metadata: bool, minify_key: bool, minify_key_len: usize, minify_key_prefix: String, @@ -54,6 +56,30 @@ impl Args { Ok(()) } + fn consume_metadata(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { + let lit_bool = input.parse::()?; + self.metadata = lit_bool.value; + // Load the config from Cargo.toml. This can be overridden by subsequent options. + if self.metadata { + // CARGO_MANIFEST_DIR is current build directory + let cargo_dir = std::env::var("CARGO_MANIFEST_DIR") + .map_err(|_| input.error("The CARGO_MANIFEST_DIR is required fo `metadata`"))?; + let current_dir = std::path::PathBuf::from(cargo_dir); + let cfg = I18nConfig::load(¤t_dir) + .map_err(|_| input.error("Failed to load config from Cargo.toml for `metadata`"))?; + self.locales_path = cfg.load_path; + self.default_locale = Some(cfg.default_locale.clone()); + if !cfg.fallback.is_empty() { + self.fallback = Some(cfg.fallback); + } + self.minify_key = cfg.minify_key; + self.minify_key_len = cfg.minify_key_len; + self.minify_key_prefix = cfg.minify_key_prefix; + self.minify_key_thresh = cfg.minify_key_thresh; + } + Ok(()) + } + fn consume_minify_key(&mut self, input: syn::parse::ParseStream) -> syn::parse::Result<()> { let lit_bool = input.parse::()?; self.minify_key = lit_bool.value; @@ -96,6 +122,9 @@ impl Args { let val = input.parse::()?; self.extend = Some(val); } + "metadata" => { + self.consume_metadata(input)?; + } "minify_key" => { self.consume_minify_key(input)?; } @@ -137,6 +166,16 @@ impl syn::parse::Parse for Args { /// # fn v4() { /// i18n!("locales", fallback = ["en", "es"]); /// # } + /// # fn v5() { + /// i18n!("locales", fallback = ["en", "es"], + /// minify_key = true, + /// minify_key_len = 12, + /// minify_key_prefix = "T.", + /// minify_key_thresh = 64); + /// # } + /// # fn v6() { + /// i18n!(metadata = true); + /// # } /// ``` /// /// Ref: https://docs.rs/syn/latest/syn/parse/index.html @@ -145,8 +184,10 @@ impl syn::parse::Parse for Args { let mut result = Self { locales_path: String::from("locales"), + default_locale: None, fallback: None, extend: None, + metadata: false, minify_key: DEFAULT_MINIFY_KEY, minify_key_len: DEFAULT_MINIFY_KEY_LEN, minify_key_prefix: DEFAULT_MINIFY_KEY_PREFIX.to_owned(), @@ -175,6 +216,7 @@ impl syn::parse::Parse for Args { /// /// - `fallback` for set the fallback locale, if present [`t!`](macro.t.html) macro will use it as the fallback locale. /// - `backend` for set the backend, if present [`t!`](macro.t.html) macro will use it as the backend. +/// - `metadata` to enable/disable loading of the [package.metadata.i18n] config from Cargo.toml, default: `false`. /// - `minify_key` for enable/disable minify key, default: [`DEFAULT_MINIFY_KEY`](constant.DEFAULT_MINIFY_KEY.html). /// - `minify_key_len` for set the minify key length, default: [`DEFAULT_MINIFY_KEY_LEN`](constant.DEFAULT_MINIFY_KEY_LEN.html), /// * The range of available values is from `0` to `24`. @@ -203,6 +245,9 @@ impl syn::parse::Parse for Args { /// minify_key_prefix = "T.", /// minify_key_thresh = 64); /// # } +/// # fn v6() { +/// i18n!(metadata = true); +/// # } /// ``` #[proc_macro] pub fn i18n(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -249,6 +294,14 @@ fn generate_code( }); }); + let default_locale = if let Some(default_locale) = args.default_locale { + quote! { + rust_i18n::set_locale(#default_locale); + } + } else { + quote! {} + }; + let fallback = if let Some(fallback) = args.fallback { quote! { Some(&[#(#fallback),*]) @@ -285,6 +338,8 @@ fn generate_code( #(#all_translations)* #extend_code + #default_locale + Box::new(backend) }); diff --git a/examples/app-metadata/Cargo.toml b/examples/app-metadata/Cargo.toml new file mode 100644 index 0000000..d6367c6 --- /dev/null +++ b/examples/app-metadata/Cargo.toml @@ -0,0 +1,18 @@ +[package] +edition = "2021" +name = "app-metadata" +version = "0.1.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rust-i18n = { path = "../.." } + +[package.metadata.i18n] +available-locales = ["en", "zh"] +default-locale = "zh" +load-path = "locales" +minify-key = true +minify-key-len = 12 +minify-key-prefix = "T." +minify-key-thresh = 4 diff --git a/examples/app-metadata/locales/v2.yml b/examples/app-metadata/locales/v2.yml new file mode 100644 index 0000000..4a2b5e8 --- /dev/null +++ b/examples/app-metadata/locales/v2.yml @@ -0,0 +1,23 @@ +_version: 2 + +T.1LokVzuiIrh1: + en: "Hello, world!" + zh: "你好,世界!" +T.53pFZEJAcwid: + en: ABCDEF + zh: 甲乙丙丁戊己 +T.6zJ2nRuJ42Z5: + en: ABCDE + zh: 甲乙丙丁戊 +a: + en: A + zh: 甲 +ab: + en: AB + zh: 甲乙 +abc: + en: ABC + zh: 甲乙丙 +abcd: + en: ABCD + zh: 甲乙丙丁 diff --git a/examples/app-metadata/src/main.rs b/examples/app-metadata/src/main.rs new file mode 100644 index 0000000..b46d066 --- /dev/null +++ b/examples/app-metadata/src/main.rs @@ -0,0 +1,24 @@ +use rust_i18n::t; + +rust_i18n::i18n!(metadata = true); + +fn main() { + let locales = rust_i18n::available_locales!(); + println!("Available locales: {:?}", locales); + println!(); + + assert_eq!(t!("a"), "甲"); + assert_eq!(t!("ab"), "甲乙"); + assert_eq!(t!("abc"), "甲乙丙"); + assert_eq!(t!("abcd"), "甲乙丙丁"); + assert_eq!(t!("abcde"), "甲乙丙丁戊"); + assert_eq!(t!("abcdef"), "甲乙丙丁戊己"); + assert_eq!(t!("Hello, world!"), "你好,世界!"); + assert_eq!(t!("a", locale = "en"), "A"); + assert_eq!(t!("ab", locale = "en"), "AB"); + assert_eq!(t!("abc", locale = "en"), "ABC"); + assert_eq!(t!("abcd", locale = "en"), "ABCD"); + assert_eq!(t!("abcde", locale = "en"), "ABCDE"); + assert_eq!(t!("abcdef", locale = "en"), "ABCDEF"); + assert_eq!(t!("Hello, world!", locale = "en"), "Hello, world!"); +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e8d86c2..8eb3e07 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -84,6 +84,10 @@ mod tests { } } + mod test5 { + rust_i18n::i18n!(metadata = true); + } + #[test] fn check_test_environment() { assert_eq!( From 17d931daff642ef314183a227c3f51d2fea2cda4 Mon Sep 17 00:00:00 2001 From: Varphone Wong Date: Mon, 29 Jan 2024 00:23:48 +0800 Subject: [PATCH 39/39] Remove unexpected files --- examples/atomic_str/Cargo.toml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 examples/atomic_str/Cargo.toml diff --git a/examples/atomic_str/Cargo.toml b/examples/atomic_str/Cargo.toml deleted file mode 100644 index 694fd19..0000000 --- a/examples/atomic_str/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "atomic_str" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -rust-i18n = { path = "../../" }