Skip to content

Commit 410adde

Browse files
authored
Reduce allocations create unnecessary HashMaps and Strings (#133)
For me, the `rust_i18n::i18n!` macro created tens of thousands of hashmaps, and allocated tens of thousands of strings. Both of which were unnecessary. This was such a problem, that it caused stack overflows (on Windows only). This PR makes 2 optimizations: - Reduce how `HashMap`s are created: from one for every pair, to one for every language. Bringing the amount of HashMaps created from 14,000 to 32 - Reduce how many `String`s are created from `&'static str` by using `Cow<'static, str>` **Optimization 1**: Don't create so many hashmaps. The expansion of `i18n!` creates a `HashMap` for every translation pair. I have a lot of translations, so for me thousands of translations were being created. Creating `HashMap` is expensive, there's no reason to do it once for every pair In this PR Ive changed the expansion so only a single HashMap is created **per language**. That brings up the count of HashMap from thousands to a only a couple. **Optimization 2**: Don't allocate so many strings. Every call to `add_translations` allocates both for the key, and each individual inner pair too. Here is the body of the `add_translations` function: ```rust pub fn add_translations(&mut self, locale: &str, data: &HashMap<&str, &str>) { let data = data .iter() .map(|(k, v)| ((*k).into(), (*v).into())) .collect::<HashMap<_, _>>(); let trs = self.translations.entry(locale.into()).or_default(); trs.extend(data); } ``` It's considered best practice to make your function signature clear. Usually, it's better to require a `String` rather than `&str` if you're just going to `.to_string()` it in the function body. it also iterates over the entire passed `data` to clone everything, and then creates **another** hashmap (the `collect`) All of this was causing `stack overflow` for me on Windows. The expansion of `i18n!` generated 14,000 HashMaps and spanned over 50,000 lines. With this PR, that's not an issue anymore. `add_translations` was changed to this: ```rust pub fn add_translations( &mut self, locale: Cow<'static, str>, data: HashMap<Cow<'static, str>, Cow<'static, str>>, ) { let trs = self.translations.entry(locale.into()).or_default(); trs.extend(data); } ``` Most inputs to it are static strings. There's no need to turn them into a `String`, so they are passed as `Cow<'static, str>`. I also added an `impl IntoIterator for SimpleBackend`. The example expansion is located at `examples/app-egui/src/main.rs` # Old expansion ```rust static _RUST_I18N_BACKEND: std::sync::LazyLock<Box<dyn rust_i18n::Backend>> = std::sync::LazyLock::new(|| { let mut backend = rust_i18n::SimpleBackend::new(); backend.add_translations( "en", &std::collections::HashMap::from([("Arthur", "Arthur")]), ); backend.add_translations("en", &std::collections::HashMap::from([("age", "age")])); ``` Note that then every of the `&'static str` you see will be `.to_string()`d in the function call # New expansion This expansion both has way less overhead, and won't cause any stack overflows ```rust static _RUST_I18N_BACKEND: std::sync::LazyLock<Box<dyn rust_i18n::Backend>> = std::sync::LazyLock::new(|| { let mut backend = rust_i18n::SimpleBackend::new(); backend .add_translations( ::std::borrow::Cow::Borrowed("en"), { let map = std::collections::HashMap::with_capacity(278usize); map.insert( ::std::borrow::Cow::Borrowed("Arthur", "Arthur"), ::std::borrow::Cow::Borrowed("age", "age"), ); ```
1 parent 94a91a7 commit 410adde

2 files changed

Lines changed: 82 additions & 54 deletions

File tree

crates/macro/src/lib.rs

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -271,15 +271,29 @@ fn generate_code(
271271
translations: BTreeMap<String, BTreeMap<String, String>>,
272272
args: Args,
273273
) -> proc_macro2::TokenStream {
274-
let mut all_translations = Vec::<proc_macro2::TokenStream>::new();
275-
276-
translations.iter().for_each(|(locale, trs)| {
277-
trs.iter().for_each(|(k, v)| {
278-
all_translations.push(quote! {
279-
backend.add_translations(#locale, &std::collections::HashMap::from([(#k, #v)]));
280-
});
281-
});
274+
let all_translations = translations.iter().map(|(locale, translation)| {
275+
let translation_length = translation.len();
276+
let translation = translation.iter().map(
277+
|(k, v)| quote! { ::std::borrow::Cow::Borrowed(#k), ::std::borrow::Cow::Borrowed(#v) },
278+
);
279+
quote! {
280+
::std::borrow::Cow::Borrowed(#locale),
281+
{
282+
let mut map = std::collections::HashMap::with_capacity(#translation_length);
283+
#(
284+
map.insert(#translation);
285+
)*
286+
map
287+
}
288+
}
282289
});
290+
let all_translations = quote! {
291+
let mut backend = rust_i18n::SimpleBackend::new();
292+
293+
#(
294+
backend.add_translations(#all_translations);
295+
)*
296+
};
283297

284298
let default_locale = if let Some(default_locale) = args.default_locale {
285299
quote! {
@@ -325,8 +339,7 @@ fn generate_code(
325339
/// [PUBLIC] This is a public API, and as an example in examples/
326340
#[allow(missing_docs)]
327341
static _RUST_I18N_BACKEND: std::sync::LazyLock<Box<dyn rust_i18n::Backend>> = std::sync::LazyLock::new(|| {
328-
let mut backend = rust_i18n::SimpleBackend::new();
329-
#(#all_translations)*
342+
#all_translations
330343
#extend_code
331344
#default_locale
332345

crates/support/src/backend.rs

Lines changed: 59 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,29 @@ where
6868
/// Simple KeyValue storage backend
6969
pub struct SimpleBackend {
7070
/// All translations key is flatten key, like `en.hello.world`
71-
translations: HashMap<String, HashMap<String, String>>,
71+
translations: HashMap<Cow<'static, str>, HashMap<Cow<'static, str>, Cow<'static, str>>>,
72+
}
73+
74+
impl
75+
FromIterator<(
76+
Cow<'static, str>,
77+
HashMap<Cow<'static, str>, Cow<'static, str>>,
78+
)> for SimpleBackend
79+
{
80+
fn from_iter<
81+
I: IntoIterator<
82+
Item = (
83+
Cow<'static, str>,
84+
HashMap<Cow<'static, str>, Cow<'static, str>>,
85+
),
86+
>,
87+
>(
88+
iter: I,
89+
) -> Self {
90+
Self {
91+
translations: iter.into_iter().collect(),
92+
}
93+
}
7294
}
7395

7496
impl SimpleBackend {
@@ -85,47 +107,40 @@ impl SimpleBackend {
85107
/// # use std::collections::HashMap;
86108
/// # use rust_i18n_support::SimpleBackend;
87109
/// # let mut backend = SimpleBackend::new();
88-
/// let mut trs = HashMap::<&str, &str>::new();
89-
/// trs.insert("hello", "Hello");
90-
/// trs.insert("foo", "Foo bar");
91-
/// backend.add_translations("en", &trs);
110+
/// let mut trs = HashMap::new();
111+
/// trs.insert("hello".into(), "Hello".into());
112+
/// trs.insert("foo".into(), "Foo bar".into());
113+
/// backend.add_translations("en".into(), trs);
92114
/// ```
93-
pub fn add_translations(&mut self, locale: &str, data: &HashMap<&str, &str>) {
94-
let data = data
95-
.iter()
96-
.map(|(k, v)| ((*k).into(), (*v).into()))
97-
.collect::<HashMap<_, _>>();
98-
115+
pub fn add_translations(
116+
&mut self,
117+
locale: Cow<'static, str>,
118+
data: HashMap<Cow<'static, str>, Cow<'static, str>>,
119+
) {
99120
let trs = self.translations.entry(locale.into()).or_default();
100121
trs.extend(data);
101122
}
102123
}
103124

104125
impl Backend for SimpleBackend {
105126
fn available_locales(&self) -> Vec<Cow<'_, str>> {
106-
let mut locales = self
107-
.translations
108-
.keys()
109-
.map(|k| Cow::from(k.as_str()))
110-
.collect::<Vec<_>>();
127+
let mut locales = self.translations.keys().cloned().collect::<Vec<_>>();
111128
locales.sort();
112129
locales
113130
}
114131

115132
fn translate(&self, locale: &str, key: &str) -> Option<Cow<'_, str>> {
116133
if let Some(trs) = self.translations.get(locale) {
117-
return trs.get(key).map(|s| Cow::from(s.as_str()));
134+
return trs.get(key).cloned();
118135
}
119136

120137
None
121138
}
122139

123140
fn messages_for_locale(&self, locale: &str) -> Option<Vec<(Cow<'_, str>, Cow<'_, str>)>> {
124-
self.translations.get(locale).map(|trs| {
125-
trs.iter()
126-
.map(|(k, v)| (Cow::from(k.as_str()), Cow::from(v.as_str())))
127-
.collect()
128-
})
141+
self.translations
142+
.get(locale)
143+
.map(|trs| trs.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
129144
}
130145
}
131146

@@ -148,15 +163,15 @@ mod tests {
148163
#[test]
149164
fn test_simple_backend() {
150165
let mut backend = SimpleBackend::new();
151-
let mut data = HashMap::<&str, &str>::new();
152-
data.insert("hello", "Hello");
153-
data.insert("foo", "Foo bar");
154-
backend.add_translations("en", &data);
166+
let mut data = HashMap::new();
167+
data.insert("hello".into(), "Hello".into());
168+
data.insert("foo".into(), "Foo bar".into());
169+
backend.add_translations("en".into(), data);
155170

156-
let mut data_cn = HashMap::<&str, &str>::new();
157-
data_cn.insert("hello", "你好");
158-
data_cn.insert("foo", "Foo 测试");
159-
backend.add_translations("zh-CN", &data_cn);
171+
let mut data_cn = HashMap::new();
172+
data_cn.insert("hello".into(), "你好".into());
173+
data_cn.insert("foo".into(), "Foo 测试".into());
174+
backend.add_translations("zh-CN".into(), data_cn);
160175

161176
assert_eq!(backend.translate("en", "hello"), Some(Cow::from("Hello")));
162177
assert_eq!(backend.translate("en", "foo"), Some(Cow::from("Foo bar")));
@@ -172,24 +187,24 @@ mod tests {
172187
#[test]
173188
fn test_combined_backend() {
174189
let mut backend = SimpleBackend::new();
175-
let mut data = HashMap::<&str, &str>::new();
176-
data.insert("hello", "Hello");
177-
data.insert("foo", "Foo bar");
178-
backend.add_translations("en", &data);
190+
let mut data = HashMap::new();
191+
data.insert("hello".into(), "Hello".into());
192+
data.insert("foo".into(), "Foo bar".into());
193+
backend.add_translations("en".into(), data);
179194

180-
let mut data_cn = HashMap::<&str, &str>::new();
181-
data_cn.insert("hello", "你好");
182-
data_cn.insert("foo", "Foo 测试");
183-
backend.add_translations("zh-CN", &data_cn);
195+
let mut data_cn = HashMap::new();
196+
data_cn.insert("hello".into(), "你好".into());
197+
data_cn.insert("foo".into(), "Foo 测试".into());
198+
backend.add_translations("zh-CN".into(), data_cn);
184199

185200
let mut backend2 = SimpleBackend::new();
186-
let mut data2 = HashMap::<&str, &str>::new();
187-
data2.insert("hello", "Hello2");
188-
backend2.add_translations("en", &data2);
201+
let mut data2 = HashMap::new();
202+
data2.insert("hello".into(), "Hello2".into());
203+
backend2.add_translations("en".into(), data2);
189204

190-
let mut data_cn2 = HashMap::<&str, &str>::new();
191-
data_cn2.insert("hello", "你好2");
192-
backend2.add_translations("zh-CN", &data_cn2);
205+
let mut data_cn2 = HashMap::new();
206+
data_cn2.insert("hello".into(), "你好2".into());
207+
backend2.add_translations("zh-CN".into(), data_cn2);
193208

194209
let combined = backend.extend(backend2);
195210
assert_eq!(combined.translate("en", "hello"), Some(Cow::from("Hello2")));

0 commit comments

Comments
 (0)