diff --git a/Cargo.toml b/Cargo.toml index e7fd38f37..2102dcd34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ members = [ "askama_hyper", "testing", ] -resolver = "2" default-members = [ "askama", @@ -23,3 +22,5 @@ default-members = [ "askama_parser", "testing", ] + +resolver = "2" diff --git a/askama/Cargo.toml b/askama/Cargo.toml index 26be2a497..08b7aa69b 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -20,6 +20,7 @@ maintenance = { status = "actively-developed" } default = ["config", "humansize", "num-traits", "urlencode"] config = ["askama_derive/config"] humansize = ["askama_derive/humansize", "dep_humansize"] +i18n = ["askama_derive/i18n", "fluent-templates", "parking_lot"] markdown = ["askama_derive/markdown", "comrak"] num-traits = ["askama_derive/num-traits", "dep_num_traits"] serde-json = ["askama_derive/serde-json", "askama_escape/json", "serde", "serde_json"] @@ -44,6 +45,8 @@ askama_escape = { version = "0.10.3", path = "../askama_escape" } comrak = { version = "0.20", optional = true, default-features = false } dep_humansize = { package = "humansize", version = "2", optional = true } dep_num_traits = { package = "num-traits", version = "0.2.6", optional = true } +fluent-templates = { version = "0.8.0", optional = true } +parking_lot = { version = "0.12.1", optional = true } percent-encoding = { version = "2.1.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } diff --git a/askama/src/i18n.rs b/askama/src/i18n.rs new file mode 100644 index 000000000..1a77aabe3 --- /dev/null +++ b/askama/src/i18n.rs @@ -0,0 +1,129 @@ +//! Module for compile time checked localization +//! +//! # Example: +//! +//! [Fluent Translation List](https://projectfluent.org/) resource file `i18n/es-MX/basic.ftl`: +//! +//! ```ftl +//! greeting = ¡Hola, { $name }! +//! ``` +//! +//! Askama HTML template `templates/example.html`: +//! +//! ```html +//!

{{ localize("greeting", name: name) }}

+//! ``` +//! +//! Rust usage: +//! ```ignore +//! use askama::i18n::{langid, Locale}; +//! use askama::Template; +//! +//! askama::i18n::load!(LOCALES); +//! +//! #[derive(Template)] +//! #[template(path = "example.html")] +//! struct ExampleTemplate<'a> { +//! #[locale] +//! loc: Locale<'a>, +//! name: &'a str, +//! } +//! +//! let template = ExampleTemplate { +//! loc: Locale::new(langid!("es-MX"), &LOCALES), +//! name: "Hilda", +//! }; +//! +//! // "

¡Hola, Hilda!

" +//! template.render().unwrap(); +//! ``` + +use std::collections::HashMap; +use std::iter::FromIterator; + +// Re-export conventiently as `askama::i18n::load!()`. +// Proc-macro crates can only export macros from their root namespace. +/// Load locales at compile time. See example above for usage. +pub use askama_derive::i18n_load as load; + +pub use fluent_templates::{self, fluent_bundle::FluentValue, fs::langid, LanguageIdentifier}; +use fluent_templates::{Loader, StaticLoader}; +use parking_lot::const_mutex; + +pub struct Locale<'a> { + loader: &'a StaticLoader, + language: LanguageIdentifier, +} + +impl Locale<'_> { + pub fn new(language: LanguageIdentifier, loader: &'static StaticLoader) -> Self { + Self { loader, language } + } + + pub fn translate<'a>( + &self, + msg_id: &str, + args: impl IntoIterator)>, + ) -> Option { + let args = HashMap::<&str, FluentValue<'_>>::from_iter(args); + let args = match args.is_empty() { + true => None, + false => Some(&args), + }; + self.loader.lookup_complete(&self.language, msg_id, args) + } +} + +/// Similar to OnceCell, but it has an additional take() function, which can only be used once, +/// and only if the instance was never dereferenced. +/// +/// The struct is only meant to be used by the [`i18n_load!()`] macro. +/// Concurrent access will deliberately panic. +/// +/// Rationale: StaticLoader cannot be cloned. +#[doc(hidden)] +pub struct Unlazy(parking_lot::Mutex>); + +enum UnlazyEnum { + Generator(Option T>), + Value(Box), +} + +impl Unlazy { + pub const fn new(f: fn() -> T) -> Self { + Self(const_mutex(UnlazyEnum::Generator(Some(f)))) + } + + pub fn take(&self) -> T { + let f = match &mut *self.0.try_lock().unwrap() { + UnlazyEnum::Generator(f) => f.take(), + _ => None, + }; + f.unwrap()() + } +} + +impl std::ops::Deref for Unlazy +where + Self: 'static, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + let data = &mut *self.0.try_lock().unwrap(); + let value: &T = match data { + UnlazyEnum::Generator(f) => { + *data = UnlazyEnum::Value(Box::new(f.take().unwrap()())); + match data { + UnlazyEnum::Value(value) => value, + _ => unreachable!(), + } + } + UnlazyEnum::Value(value) => value, + }; + + // SAFETY: This transmutation is safe because once a value is assigned, + // it won't be unassigned again, and Self has static lifetime. + unsafe { std::mem::transmute(value) } + } +} diff --git a/askama/src/lib.rs b/askama/src/lib.rs index a98989e85..6ff1d225f 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -59,13 +59,15 @@ //! in the configuration file. The default syntax , "default", is the one //! provided by Askama. -#![forbid(unsafe_code)] +#![cfg_attr(not(feature = "i18n"), forbid(unsafe_code))] #![deny(elided_lifetimes_in_paths)] #![deny(unreachable_pub)] mod error; pub mod filters; pub mod helpers; +#[cfg(feature = "i18n")] +pub mod i18n; use std::fmt; diff --git a/askama_derive/Cargo.toml b/askama_derive/Cargo.toml index c31130293..70c929c20 100644 --- a/askama_derive/Cargo.toml +++ b/askama_derive/Cargo.toml @@ -16,11 +16,12 @@ proc-macro = true [features] config = ["serde", "basic-toml"] humansize = [] +i18n = ["fluent-syntax", "fluent-templates", "serde", "basic-toml", "parser/i18n"] markdown = [] -urlencode = [] +num-traits = [] serde-json = [] serde-yaml = [] -num-traits = [] +urlencode = [] with-actix-web = [] with-axum = [] with-gotham = [] @@ -31,6 +32,8 @@ with-tide = [] with-warp = [] [dependencies] +fluent-syntax = { version = "0.11.0", optional = true, default-features = false } +fluent-templates = { version = "0.8.0", optional = true, default-features = false } parser = { package = "askama_parser", version = "0.2", path = "../askama_parser" } mime = "0.3" mime_guess = "2" diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index df15bdc49..aab094fbe 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1154,10 +1154,43 @@ impl<'a> Generator<'a> { Expr::RustMacro(ref path, args) => self.visit_rust_macro(buf, path, args), Expr::Try(ref expr) => self.visit_try(buf, expr.as_ref())?, Expr::Tuple(ref exprs) => self.visit_tuple(buf, exprs)?, + #[cfg(feature = "i18n")] + Expr::Localize(ref msg_id, ref args) => self.visit_localize(buf, msg_id, args)?, Expr::NamedArgument(_, ref expr) => self.visit_named_argument(buf, expr)?, }) } + #[cfg(feature = "i18n")] + fn visit_localize( + &mut self, + buf: &mut Buffer, + msg_id: &Expr<'_>, + args: &[(&str, Expr<'_>)], + ) -> Result { + let localizer = + self.input.localizer.as_deref().ok_or( + "You need to annotate a field with #[locale] to use the localize() function.", + )?; + + buf.write(&format!( + "self.{}.translate(", + normalize_identifier(localizer) + )); + self.visit_expr(buf, msg_id)?; + buf.writeln(", [")?; + buf.indent(); + for (k, v) in args { + buf.write(&format!("({:?}, ::askama::i18n::FluentValue::from(", k)); + self.visit_expr(buf, v)?; + buf.writeln(")),")?; + } + buf.dedent()?; + // Safe to unwrap, as `msg_id` is checked at compile time. + buf.write("]).unwrap()"); + + Ok(DisplayWrap::Unwrapped) + } + fn visit_try( &mut self, buf: &mut Buffer, @@ -1984,6 +2017,10 @@ pub(crate) fn is_cacheable(expr: &Expr<'_>) -> bool { Expr::Group(arg) => is_cacheable(arg), Expr::Tuple(args) => args.iter().all(is_cacheable), Expr::NamedArgument(_, expr) => is_cacheable(expr), + #[cfg(feature = "i18n")] + Expr::Localize(msg_id, args) => { + is_cacheable(msg_id) && args.iter().all(|(_, arg)| is_cacheable(arg)) + } // We have too little information to tell if the expression is pure: Expr::Call(_, _) => false, Expr::RustMacro(_, _) => false, diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs new file mode 100644 index 000000000..3c06c5e9a --- /dev/null +++ b/askama_derive/src/i18n.rs @@ -0,0 +1,387 @@ +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::fs::{DirEntry, OpenOptions}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use basic_toml::from_str; +use fluent_syntax::ast::{ + Expression, InlineExpression, PatternElement, Resource, Variant, VariantKey, +}; +use fluent_syntax::parser::parse_runtime; +use fluent_templates::lazy_static::lazy_static; +use fluent_templates::loader::build_fallbacks; +use fluent_templates::LanguageIdentifier; +use proc_macro::TokenStream; +use proc_macro2::{Ident, TokenStream as TokenStream2}; +use quote::quote_spanned; +use serde::Deserialize; +use syn::parse::{Parse, ParseStream}; +use syn::spanned::Spanned; +use syn::{parse2, Visibility}; + +use crate::CompileError; + +type FileResource = (PathBuf, Resource); + +macro_rules! mk_static { + ($(let $ident:ident: $ty:ty = $expr:expr;)*) => { + $( + let $ident = { + let value: Option<$ty> = Some($expr); + unsafe { + static mut VALUE: Option<$ty> = None; + VALUE = value; + match &VALUE { + Some(value) => value, + None => unreachable!(), + } + } + }; + )* + }; +} + +struct Variable { + vis: Visibility, + name: Ident, +} + +impl Parse for Variable { + fn parse(input: ParseStream<'_>) -> syn::Result { + let vis = input.parse().unwrap_or(Visibility::Inherited); + let name = input.parse()?; + Ok(Variable { vis, name }) + } +} + +struct Configuration { + pub(crate) fallback: LanguageIdentifier, + pub(crate) use_isolating: bool, + pub(crate) core_locales: Option, + pub(crate) locales: Vec<(LanguageIdentifier, Vec)>, + pub(crate) fallbacks: &'static HashMap>, + pub(crate) assets_dir: PathBuf, +} + +#[derive(Default, Deserialize)] +struct I18nConfig { + #[serde(default)] + pub(crate) fallback_language: Option, + #[serde(default)] + pub(crate) fluent: Option, +} + +#[derive(Default, Deserialize)] +struct I18nFluent { + #[serde(default)] + pub(crate) assets_dir: Option, + #[serde(default)] + pub(crate) core_locales: Option, + #[serde(default)] + pub(crate) use_isolating: Option, +} + +fn format_err(path: &Path, err: impl Display) -> String { + format!("error processing {:?}: {}", path, err) +} + +fn read_resource(path: PathBuf) -> Result { + let mut buf = String::new(); + OpenOptions::new() + .read(true) + .open(&path) + .map_err(|err| format_err(&path, err))? + .read_to_string(&mut buf) + .map_err(|err| format_err(&path, err))?; + + let resource = match parse_runtime(buf) { + Ok(resource) => resource, + Err((_, err_vec)) => return Err(format_err(&path, err_vec.first().unwrap())), + }; + Ok((path, resource)) +} + +fn read_lang_dir( + entry: Result, +) -> Result)>, String> { + let entry = match entry { + Ok(entry) => entry, + Err(_) => return Ok(None), + }; + + let language = entry + .file_name() + .to_str() + .and_then(|s| LanguageIdentifier::from_str(s).ok()); + let language: LanguageIdentifier = match language { + Some(language) => language, + None => return Ok(None), + }; + + let dir_iter = match entry.path().read_dir() { + Ok(dir_iter) => dir_iter, + Err(_) => return Ok(None), + }; + + let mut resources = vec![]; + for entry in dir_iter.flatten() { + let path = entry.path(); + if path.to_str().map(|s| s.ends_with(".ftl")).unwrap_or(false) { + resources.push(read_resource(path)?); + }; + } + if resources.is_empty() { + return Ok(None); + } + + resources.sort_by(|(l, _), (r, _)| Path::cmp(l, r)); + Ok(Some((language, resources))) +} + +fn read_configuration() -> Result { + let root = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); + let root = root.canonicalize().unwrap_or(root); + + let i18n_toml = root.join("i18n.toml"); + let config = match i18n_toml.exists() { + false => I18nConfig::default(), + true => { + let mut buf = String::new(); + OpenOptions::new() + .read(true) + .open(&i18n_toml) + .map_err(|err| format_err(&i18n_toml, err))? + .read_to_string(&mut buf) + .map_err(|err| format_err(&i18n_toml, err))?; + from_str(&buf).map_err(|err| format_err(&i18n_toml, err))? + } + }; + let fluent = config.fluent.unwrap_or_default(); + + let fallback = config.fallback_language.as_deref().unwrap_or("en"); + let fallback: LanguageIdentifier = match fallback.parse() { + Ok(fallback) => fallback, + Err(err) => { + return Err(format!( + "not a valid LanguageIdentifier {:?} for fallback_language: {}", + err, fallback, + )) + } + }; + + let core_locales = match fluent.core_locales { + Some(path) => { + let path = match path.is_absolute() { + true => path, + false => root.join(path), + }; + if path.to_str().is_none() { + return Err(format!( + "core_locales path contains illegal UTF-8 characters: {:?}", + path, + )); + }; + Some(read_resource(path)?) + } + None => None, + }; + + let assets_dir = match fluent.assets_dir { + Some(path) if path.is_absolute() => todo!(), + Some(path) => root.join(&path), + None => root.join("i18n"), + }; + let mut locales = { + let mut locales = vec![]; + for entry in assets_dir + .read_dir() + .map_err(|err| format_err(&assets_dir, err))? + { + if let Some(datum) = read_lang_dir(entry)? { + locales.push(datum); + } + } + locales + }; + locales.sort_by(|(l1, _), (l2, _)| LanguageIdentifier::cmp(l1, l2)); + + mk_static! { + let locales_: Vec = locales.iter().map(|(l, _)| l.clone()).collect(); + let fallbacks: HashMap> = build_fallbacks( + locales_, + ); + }; + + Ok(Configuration { + fallback, + use_isolating: fluent.use_isolating.unwrap_or(false), + core_locales, + locales, + fallbacks, + assets_dir, + }) +} + +fn get_i18n_config() -> Result<&'static Configuration, CompileError> { + lazy_static! { + static ref CONFIGURATION: Result = read_configuration(); + } + match &*CONFIGURATION { + Ok(configuration) => Ok(configuration), + Err(err) => Err(err.as_str().into()), + } +} + +pub(crate) fn load(input: TokenStream) -> Result { + let configuration = get_i18n_config()?; + + let input: TokenStream2 = input.into(); + let span = input.span(); + let variable: Variable = match parse2(input) { + Ok(variable) => variable, + Err(err) => return Err(format!("could not parse i18n_load!(…): {}", err).into()), + }; + + let vis = variable.vis; + let name = variable.name; + let assets_dir = configuration.assets_dir.to_str().unwrap(); + let fallback = configuration.fallback.to_string(); + let core_locales = configuration.core_locales.as_ref().map(|(s, _)| { + let s = s.to_str().unwrap(); + quote_spanned!(span => core_locales: #s,) + }); + let customise = match configuration.use_isolating { + false => Some(quote_spanned!(span => customise: |b| b.set_use_isolating(false),)), + true => None, + }; + + let ts = quote_spanned! { + span => + #vis static #name: + ::askama::i18n::fluent_templates::once_cell::sync::Lazy::< + ::askama::i18n::fluent_templates::StaticLoader + > = ::askama::i18n::fluent_templates::once_cell::sync::Lazy::new(|| { + mod fluent_templates { + // RATIONALE: the user might not use fluent_templates directly. + pub use ::askama::i18n::fluent_templates::*; + pub mod once_cell { + pub mod sync { + pub use ::askama::i18n::Unlazy as Lazy; + } + } + } + ::askama::i18n::fluent_templates::static_loader! { + pub static LOCALES = { + locales: #assets_dir, + fallback_language: #fallback, + #core_locales + #customise + }; + } + LOCALES.take() + }); + }; + Ok(ts.into()) +} + +pub(crate) fn arguments_of(msg_id: &str) -> Result, CompileError> { + let config = get_i18n_config()?; + let entry = config.fallbacks[&config.fallback] + .iter() + .filter_map(|l1| { + config + .locales + .binary_search_by(|(l2, _)| LanguageIdentifier::cmp(l2, l1)) + .ok() + }) + .flat_map(|index| &config.locales[index].1) + .chain(config.core_locales.iter()) + .flat_map(|(_, resource)| &resource.body) + .filter_map(|entry| match entry { + fluent_syntax::ast::Entry::Message(entry) => Some(entry), + _ => None, + }) + .find(|entry| entry.id.name == msg_id) + .ok_or_else(|| CompileError::from(format!("msg_id {:?} not found", msg_id)))?; + + let keys = entry + .value + .iter() + .flat_map(|v| v.elements.iter()) + .filter_map(|p| match p { + PatternElement::Placeable { expression } => Some(expression), + _ => None, + }) + .flat_map(expr_to_key) + .collect(); + Ok(keys) +} + +fn expr_to_key(expr: &'static Expression) -> Vec<&'static str> { + let (selector, variants): (&InlineExpression, &[Variant]) = match expr { + Expression::Select { selector, variants } => (selector, variants), + Expression::Inline(selector) => (selector, &[]), + }; + + let variant_keys = variants.iter().filter_map(|v| match &v.key { + VariantKey::Identifier { name } => Some(name.as_str()), + _ => None, + }); + + let variant_values = variants + .iter() + .flat_map(|v| v.value.elements.iter()) + .filter_map(|v| match v { + PatternElement::Placeable { expression } => Some(expression), + _ => None, + }) + .flat_map(expr_to_key); + + let selector_keys = inline_expr_to_key(selector); + + let mut v = vec![]; + v.extend(variant_keys); + v.extend(variant_values); + v.extend(selector_keys); + v +} + +fn inline_expr_to_key(selector: &'static InlineExpression) -> Vec<&'static str> { + let mut v = vec![]; + v.extend(selector_placeable(selector)); + v.extend(selector_variable(selector)); + v.extend(selector_function(selector)); + v +} + +fn selector_placeable(e: &'static InlineExpression) -> impl Iterator { + let e = match e { + InlineExpression::Placeable { expression } => Some(expression), + _ => None, + }; + e.into_iter().flat_map(|e| expr_to_key(e)) +} + +fn selector_variable(e: &'static InlineExpression) -> impl Iterator { + let id = match e { + InlineExpression::VariableReference { id } => Some(id.name.as_str()), + _ => None, + }; + id.into_iter() +} + +fn selector_function(e: &'static InlineExpression) -> impl Iterator { + let arguments = match e { + InlineExpression::FunctionReference { arguments, .. } => Some(arguments), + _ => None, + }; + arguments.into_iter().flat_map(|a| { + a.named + .iter() + .map(|n| &n.value) + .chain(&a.positional) + .flat_map(inline_expr_to_key) + }) +} diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index d10fd7fea..98462d2d1 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -20,6 +20,8 @@ pub(crate) struct TemplateInput<'a> { pub(crate) ext: Option<&'a str>, pub(crate) mime_type: String, pub(crate) path: PathBuf, + #[cfg(feature = "i18n")] + pub(crate) localizer: Option, } impl TemplateInput<'_> { @@ -54,6 +56,42 @@ impl TemplateInput<'_> { } }; + let localizer = match ast.data { + syn::Data::Struct(syn::DataStruct { + fields: syn::Fields::Named(ref fields), + .. + }) => { + let mut localizers = + fields + .named + .iter() + .filter(|&f| f.ident.is_some()) + .flat_map( + |f| match f.attrs.iter().any(|a| a.path().is_ident("locale")) { + true => Some(f.ident.as_ref()?.to_string()), + false => None, + }, + ); + match localizers.next() { + Some(localizer) => { + if !cfg!(feature = "i18n") { + return Err( + "You need to activate the \"i18n\" feature to use #[locale]." + .into(), + ); + } else if localizers.next().is_some() { + return Err("You cannot mark more than one field as #[locale].".into()); + } + Some(localizer) + } + None => None, + } + } + _ => None, + }; + #[cfg(not(feature = "i18n"))] + drop(localizer); + // Validate syntax let syntax = syntax.as_deref().map_or_else( || Ok(config.syntaxes.get(config.default_syntax).unwrap()), @@ -97,6 +135,8 @@ impl TemplateInput<'_> { ext: ext.as_deref(), mime_type, path, + #[cfg(feature = "i18n")] + localizer, }) } diff --git a/askama_derive/src/lib.rs b/askama_derive/src/lib.rs index 65160c51c..5708f2b52 100644 --- a/askama_derive/src/lib.rs +++ b/askama_derive/src/lib.rs @@ -1,3 +1,4 @@ +#![cfg_attr(not(feature = "i18n"), forbid(unsafe_code))] #![deny(elided_lifetimes_in_paths)] #![deny(unreachable_pub)] @@ -14,11 +15,13 @@ use config::Config; mod generator; use generator::{Generator, MapChain}; mod heritage; +#[cfg(feature = "i18n")] +mod i18n; use heritage::{Context, Heritage}; mod input; use input::{Print, TemplateArgs, TemplateInput}; -#[proc_macro_derive(Template, attributes(template))] +#[proc_macro_derive(Template, attributes(template, locale))] pub fn derive_template(input: TokenStream) -> TokenStream { let ast = syn::parse::(input).unwrap(); match build_template(&ast) { @@ -70,6 +73,18 @@ pub(crate) fn build_template(ast: &syn::DeriveInput) -> Result TokenStream { + #[cfg(feature = "i18n")] + match i18n::load(_input) { + Ok(ts) => ts, + Err(err) => err.into_compile_error(), + } + + #[cfg(not(feature = "i18n"))] + CompileError::from(r#"Activate the "i18n" feature to use i18n_load!()."#).into_compile_error() +} + #[derive(Debug, Clone)] struct CompileError { msg: Cow<'static, str>, diff --git a/askama_parser/Cargo.toml b/askama_parser/Cargo.toml index eecf32e40..3763806f4 100644 --- a/askama_parser/Cargo.toml +++ b/askama_parser/Cargo.toml @@ -13,5 +13,14 @@ readme = "README.md" edition = "2021" rust-version = "1.65" +[features] +i18n = ["fluent-syntax", "fluent-templates", "serde", "basic-toml"] + [dependencies] nom = { version = "7", default-features = false, features = ["alloc"] } +fluent-syntax = { version = "0.11.0", default-features = false, optional = true } +fluent-templates = { version = "0.8.0", default-features = false, optional = true } +serde = { version = "1.0.193", default-features = false, features = ["derive"], optional = true } +basic-toml = { version = "0.1.7", default-features = false, optional = true } +syn = { version = "2.0.43", default-features = false } +proc-macro2 = { version = "1.0.71", default-features = false } diff --git a/askama_parser/src/expr.rs b/askama_parser/src/expr.rs index 65a783553..6f3c50748 100644 --- a/askama_parser/src/expr.rs +++ b/askama_parser/src/expr.rs @@ -72,6 +72,8 @@ pub enum Expr<'a> { Call(Box>, Vec>), RustMacro(Vec<&'a str>, &'a str), Try(Box>), + #[cfg(feature = "i18n")] + Localize(Box>, Vec<(&'a str, Expr<'a>)>), } impl<'a> Expr<'a> { @@ -161,6 +163,7 @@ impl<'a> Expr<'a> { )(i) }; alt(( + Self::localize, map(range_right, |(op, right)| { Self::Range(op, None, right.map(Box::new)) }), @@ -300,6 +303,74 @@ impl<'a> Expr<'a> { fn char(i: &'a str) -> ParseResult<'a, Self> { map(char_lit, Self::CharLit)(i) } + + #[cfg(not(feature = "i18n"))] + fn localize(i: &'a str) -> ParseResult<'a, Self> { + let (i, _) = pair(tag("localize"), ws(tag("(")))(i)?; + eprintln!(r#"Activate the "i18n" feature to use {{ localize() }}."#); + Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))) + } + + #[cfg(feature = "i18n")] + fn localize(i: &'a str) -> ParseResult<'a, Self> { + fn localize_args<'a>(mut i: &'a str) -> ParseResult<'a, Vec<(&str, Expr<'_>)>> { + let mut args = Vec::<(&str, Expr<'_>)>::new(); + + let mut p = opt(tuple((ws(tag(",")), identifier, ws(tag(":")), move |i| { + Expr::parse(i, Level(0)) + }))); + while let (j, Some((_, k, _, v))) = p(i)? { + if args.iter().any(|&(a, _)| a == k) { + eprintln!("Duplicated key: {:?}", k); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } + + args.push((k, v)); + i = j; + } + + let (i, _) = opt(tag(","))(i)?; + Ok((i, args)) + } + + let (j, (_, _, (msg_id, args, _))) = tuple(( + tag("localize"), + ws(tag("(")), + cut(tuple(( + move |i| Expr::parse(i, Level(0)), + localize_args, + ws(tag(")")), + ))), + ))(i)?; + + if let Expr::StrLit(msg_id) = msg_id { + let mut msg_args = match crate::i18n::arguments_of(msg_id) { + Ok(args) => args, + Err(err) => { + eprintln!("{}", err.msg); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } + }; + for &(call_arg, _) in &args { + if !msg_args.remove(call_arg) { + eprintln!( + "Fluent template {:?} does not contain argument {:?}", + msg_id, call_arg, + ); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } + } + if !msg_args.is_empty() { + eprintln!( + "Missing argument(s) {:?} to fluent template {:?}", + msg_args, msg_id, + ); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } + } + + Ok((j, Expr::Localize(msg_id.into(), args))) + } } enum Suffix<'a> { diff --git a/askama_parser/src/i18n.rs b/askama_parser/src/i18n.rs new file mode 100644 index 000000000..4dbface77 --- /dev/null +++ b/askama_parser/src/i18n.rs @@ -0,0 +1,425 @@ +use fluent_templates::LanguageIdentifier; +use std::borrow::Cow; +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::fs::{DirEntry, OpenOptions}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use basic_toml::from_str; +use fluent_syntax::ast::{ + Expression, InlineExpression, PatternElement, Resource, Variant, VariantKey, +}; +use fluent_syntax::parser::parse_runtime; +use fluent_templates::loader::build_fallbacks; +use proc_macro2::{Ident, TokenStream}; +//use quote::quote_spanned; +use proc_macro2::Span; +use serde::Deserialize; +//use syn::parse::{Parse, ParseStream}; +//use syn::spanned::Spanned; +//use syn::{parse2, Visibility}; +// +//use crate::CompileError; +// +type FileResource = (PathBuf, Resource); +// +macro_rules! mk_static { + ($(let $ident:ident: $ty:ty = $expr:expr;)*) => { + $( + let $ident = { + let value: Option<$ty> = Some($expr); + unsafe { + static mut VALUE: Option<$ty> = None; + VALUE = value; + match &VALUE { + Some(value) => value, + None => unreachable!(), + } + } + }; + )* + }; +} +// +//struct Variable { +// vis: Visibility, +// name: Ident, +//} +// +//impl Parse for Variable { +// fn parse(input: ParseStream<'_>) -> syn::Result { +// let vis = input.parse().unwrap_or(Visibility::Inherited); +// let name = input.parse()?; +// Ok(Variable { vis, name }) +// } +//} +// + +#[derive(Debug, Clone)] +pub(crate) struct CompileError { + pub(crate) msg: Cow<'static, str>, + pub(crate) span: Span, +} + +impl From<&'static str> for CompileError { + #[inline] + fn from(s: &'static str) -> Self { + Self::new(s, Span::call_site()) + } +} +impl From for CompileError { + #[inline] + fn from(s: String) -> Self { + Self::new(s, Span::call_site()) + } +} + +impl CompileError { + fn new>>(s: S, span: Span) -> Self { + Self { + msg: s.into(), + span, + } + } + + fn into_compile_error(self) -> TokenStream { + syn::Error::new(self.span, self.msg) + .to_compile_error() + .into() + } +} +struct Configuration { + pub(crate) fallback: LanguageIdentifier, + pub(crate) use_isolating: bool, + pub(crate) core_locales: Option, + pub(crate) locales: Vec<(LanguageIdentifier, Vec)>, + pub(crate) fallbacks: &'static HashMap>, + pub(crate) assets_dir: PathBuf, +} + +#[derive(Default, Deserialize)] +struct I18nConfig { + #[serde(default)] + pub(crate) fallback_language: Option, + #[serde(default)] + pub(crate) fluent: Option, +} + +#[derive(Default, Deserialize)] +struct I18nFluent { + #[serde(default)] + pub(crate) assets_dir: Option, + #[serde(default)] + pub(crate) core_locales: Option, + #[serde(default)] + pub(crate) use_isolating: Option, +} + +fn format_err(path: &Path, err: impl Display) -> String { + format!("error processing {:?}: {}", path, err) +} +// +fn read_resource(path: PathBuf) -> Result { + let mut buf = String::new(); + OpenOptions::new() + .read(true) + .open(&path) + .map_err(|err| format_err(&path, err))? + .read_to_string(&mut buf) + .map_err(|err| format_err(&path, err))?; + + let resource = match parse_runtime(buf) { + Ok(resource) => resource, + Err((_, err_vec)) => return Err(format_err(&path, err_vec.first().unwrap())), + }; + Ok((path, resource)) +} + +fn read_lang_dir( + entry: Result, +) -> Result)>, String> { + let entry = match entry { + Ok(entry) => entry, + Err(_) => return Ok(None), + }; + + let language = entry + .file_name() + .to_str() + .and_then(|s| LanguageIdentifier::from_str(s).ok()); + let language: LanguageIdentifier = match language { + Some(language) => language, + None => return Ok(None), + }; + + let dir_iter = match entry.path().read_dir() { + Ok(dir_iter) => dir_iter, + Err(_) => return Ok(None), + }; + + let mut resources = vec![]; + for entry in dir_iter.flatten() { + let path = entry.path(); + if path.to_str().map(|s| s.ends_with(".ftl")).unwrap_or(false) { + resources.push(read_resource(path)?); + }; + } + if resources.is_empty() { + return Ok(None); + } + + resources.sort_by(|(l, _), (r, _)| Path::cmp(l, r)); + Ok(Some((language, resources))) +} + +fn read_configuration() -> Result { + let root = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); + let root = root.canonicalize().unwrap_or(root); + + let i18n_toml = root.join("i18n.toml"); + let config = match i18n_toml.exists() { + false => I18nConfig::default(), + true => { + let mut buf = String::new(); + OpenOptions::new() + .read(true) + .open(&i18n_toml) + .map_err(|err| format_err(&i18n_toml, err))? + .read_to_string(&mut buf) + .map_err(|err| format_err(&i18n_toml, err))?; + from_str(&buf).map_err(|err| format_err(&i18n_toml, err))? + } + }; + let fluent = config.fluent.unwrap_or_default(); + + let fallback = config.fallback_language.as_deref().unwrap_or("en"); + let fallback: LanguageIdentifier = match fallback.parse() { + Ok(fallback) => fallback, + Err(err) => { + return Err(format!( + "not a valid LanguageIdentifier {:?} for fallback_language: {}", + err, fallback, + )) + } + }; + + let core_locales = match fluent.core_locales { + Some(path) => { + let path = match path.is_absolute() { + true => path, + false => root.join(path), + }; + if path.to_str().is_none() { + return Err(format!( + "core_locales path contains illegal UTF-8 characters: {:?}", + path, + )); + }; + Some(read_resource(path)?) + } + None => None, + }; + + let assets_dir = match fluent.assets_dir { + Some(path) if path.is_absolute() => todo!(), + Some(path) => root.join(&path), + None => root.join("i18n"), + }; + let mut locales = { + let mut locales = vec![]; + for entry in assets_dir + .read_dir() + .map_err(|err| format_err(&assets_dir, err))? + { + if let Some(datum) = read_lang_dir(entry)? { + locales.push(datum); + } + } + locales + }; + locales.sort_by(|(l1, _), (l2, _)| LanguageIdentifier::cmp(l1, l2)); + + mk_static! { + let locales_: Vec = locales.iter().map(|(l, _)| l.clone()).collect(); + let fallbacks: HashMap> = build_fallbacks( + locales_, + ); + }; + + Ok(Configuration { + fallback, + use_isolating: fluent.use_isolating.unwrap_or(false), + core_locales, + locales, + fallbacks, + assets_dir, + }) +} + +use fluent_templates::lazy_static::lazy_static; + +fn get_i18n_config() -> Result<&'static Configuration, CompileError> { + lazy_static! { + static ref CONFIGURATION: Result = read_configuration(); + } + match &*CONFIGURATION { + Ok(configuration) => Ok(configuration), + Err(err) => Err(err.as_str().into()), + } +} +// +//pub(crate) fn load(input: TokenStream) -> Result { +// let configuration = get_i18n_config()?; +// +// let input: TokenStream2 = input.into(); +// let span = input.span(); +// let variable: Variable = match parse2(input) { +// Ok(variable) => variable, +// Err(err) => return Err(format!("could not parse i18n_load!(…): {}", err).into()), +// }; +// +// let vis = variable.vis; +// let name = variable.name; +// let assets_dir = configuration.assets_dir.to_str().unwrap(); +// let fallback = configuration.fallback.to_string(); +// let core_locales = configuration.core_locales.as_ref().map(|(s, _)| { +// let s = s.to_str().unwrap(); +// quote_spanned!(span => core_locales: #s,) +// }); +// let customise = match configuration.use_isolating { +// false => Some(quote_spanned!(span => customise: |b| b.set_use_isolating(false),)), +// true => None, +// }; +// +// let ts = quote_spanned! { +// span => +// #vis static #name: +// ::askama::i18n::fluent_templates::once_cell::sync::Lazy::< +// ::askama::i18n::fluent_templates::StaticLoader +// > = ::askama::i18n::fluent_templates::once_cell::sync::Lazy::new(|| { +// mod fluent_templates { +// // RATIONALE: the user might not use fluent_templates directly. +// pub use ::askama::i18n::fluent_templates::*; +// pub mod once_cell { +// pub mod sync { +// pub use ::askama::i18n::Unlazy as Lazy; +// } +// } +// } +// ::askama::i18n::fluent_templates::static_loader! { +// pub static LOCALES = { +// locales: #assets_dir, +// fallback_language: #fallback, +// #core_locales +// #customise +// }; +// } +// LOCALES.take() +// }); +// }; +// Ok(ts.into()) +//} +// + +pub(crate) fn arguments_of(msg_id: &str) -> Result, CompileError> { + let config = get_i18n_config()?; + let entry = config.fallbacks[&config.fallback] + .iter() + .filter_map(|l1| { + config + .locales + .binary_search_by(|(l2, _)| LanguageIdentifier::cmp(l2, l1)) + .ok() + }) + .flat_map(|index| &config.locales[index].1) + .chain(config.core_locales.iter()) + .flat_map(|(_, resource)| &resource.body) + .filter_map(|entry| match entry { + fluent_syntax::ast::Entry::Message(entry) => Some(entry), + _ => None, + }) + .find(|entry| entry.id.name == msg_id) + .ok_or_else(|| CompileError::from(format!("msg_id {:?} not found", msg_id)))?; + + let keys = entry + .value + .iter() + .flat_map(|v| v.elements.iter()) + .filter_map(|p| match p { + PatternElement::Placeable { expression } => Some(expression), + _ => None, + }) + .flat_map(expr_to_key) + .collect(); + Ok(keys) +} + +fn expr_to_key(expr: &'static Expression) -> Vec<&'static str> { + let (selector, variants): (&InlineExpression, &[Variant]) = match expr { + Expression::Select { selector, variants } => (selector, variants), + Expression::Inline(selector) => (selector, &[]), + }; + + let variant_keys = variants.iter().filter_map(|v| match &v.key { + VariantKey::Identifier { name } => Some(name.as_str()), + _ => None, + }); + + let variant_values = variants + .iter() + .flat_map(|v| v.value.elements.iter()) + .filter_map(|v| match v { + PatternElement::Placeable { expression } => Some(expression), + _ => None, + }) + .flat_map(expr_to_key); + + let selector_keys = inline_expr_to_key(selector); + + let mut v = vec![]; + v.extend(variant_keys); + v.extend(variant_values); + v.extend(selector_keys); + v +} + +fn inline_expr_to_key(selector: &'static InlineExpression) -> Vec<&'static str> { + let mut v = vec![]; + v.extend(selector_placeable(selector)); + v.extend(selector_variable(selector)); + v.extend(selector_function(selector)); + v +} + +fn selector_placeable(e: &'static InlineExpression) -> impl Iterator { + let e = match e { + InlineExpression::Placeable { expression } => Some(expression), + _ => None, + }; + e.into_iter().flat_map(|e| expr_to_key(e)) +} + +fn selector_variable(e: &'static InlineExpression) -> impl Iterator { + let id = match e { + InlineExpression::VariableReference { id } => Some(id.name.as_str()), + _ => None, + }; + id.into_iter() +} + +fn selector_function(e: &'static InlineExpression) -> impl Iterator { + let arguments = match e { + InlineExpression::FunctionReference { arguments, .. } => Some(arguments), + _ => None, + }; + arguments.into_iter().flat_map(|a| { + a.named + .iter() + .map(|n| &n.value) + .chain(&a.positional) + .flat_map(inline_expr_to_key) + }) +} diff --git a/askama_parser/src/lib.rs b/askama_parser/src/lib.rs index 4d27d7028..2636568be 100644 --- a/askama_parser/src/lib.rs +++ b/askama_parser/src/lib.rs @@ -15,6 +15,8 @@ use nom::sequence::{delimited, pair, preceded, terminated, tuple}; use nom::{error_position, AsChar, InputTakeAtPosition}; pub mod expr; +#[cfg(feature = "i18n")] +mod i18n; pub use expr::Expr; pub mod node; pub use node::Node; diff --git a/askama_parser/src/tests.rs b/askama_parser/src/tests.rs index 3a7b4523c..811ebccd3 100644 --- a/askama_parser/src/tests.rs +++ b/askama_parser/src/tests.rs @@ -706,6 +706,74 @@ fn test_missing_space_after_kw() { )); } +#[cfg(feature = "i18n")] +#[test] +fn test_parse_localize() { + macro_rules! map { + ($($k:expr => $v:expr),* $(,)?) => {{ + use std::iter::{Iterator, IntoIterator}; + Iterator::collect(IntoIterator::into_iter([$(($k, $v),)*])) + }}; + } + assert_eq!( + super::parse(r#"{{ localize(1, v: 32 + 7) }}"#, &Syntax::default()).unwrap(), + vec![Node::Expr( + Ws(None, None), + Expr::Localize( + Expr::NumLit("1").into(), + map!( + "v" => { + Expr::BinOp("+", Expr::NumLit("32").into(), Expr::NumLit("7").into()) + } + ), + ) + )], + ); + assert_eq!( + super::parse( + r#"{{ localize(1, b: "b", c: "c", d: "d") }}"#, + &Syntax::default(), + ) + .unwrap(), + vec![Node::Expr( + Ws(None, None), + Expr::Localize( + Expr::NumLit("1").into(), + map!( + "b" => Expr::StrLit("b"), + "c" => Expr::StrLit("c"), + "d" => Expr::StrLit("d"), + ), + ) + )], + ); + assert_eq!( + super::parse( + r#"{{ localize(1, v: localize(2, v: 32 + 7) ) }}"#, + &Syntax::default(), + ) + .unwrap(), + vec![Node::Expr( + Ws(None, None), + Expr::Localize( + Expr::NumLit("1").into(), + map!( + "v" => Expr::Localize( + Expr::NumLit("2").into(), + map!( + "v" => Expr::BinOp( + "+", + Expr::NumLit("32").into(), + Expr::NumLit("7").into(), + ), + ), + ), + ), + ), + )], + ); +} + #[test] fn test_parse_array() { let syntax = Syntax::default(); diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 793de53a2..cda573835 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -8,9 +8,10 @@ rust-version = "1.65" publish = false [features] -default = ["serde-json", "markdown"] -serde-json = ["serde_json", "askama/serde-json"] +default = ["serde-json", "markdown", "i18n"] +i18n = ["askama/i18n"] markdown = ["comrak", "askama/markdown"] +serde-json = ["serde_json", "askama/serde-json"] [dependencies] askama = { path = "../askama", version = "0.12" } diff --git a/testing/i18n-basic/en-US/basic.ftl b/testing/i18n-basic/en-US/basic.ftl new file mode 100644 index 000000000..c1cc18561 --- /dev/null +++ b/testing/i18n-basic/en-US/basic.ftl @@ -0,0 +1,3 @@ +greeting = Hello, { $name }! +age = You are { $hours } hours old. +test = This is a test diff --git a/testing/i18n-basic/es-MX/basic.ftl b/testing/i18n-basic/es-MX/basic.ftl new file mode 100644 index 000000000..079ac28f5 --- /dev/null +++ b/testing/i18n-basic/es-MX/basic.ftl @@ -0,0 +1,2 @@ +greeting = ¡Hola, { $name }! +age = Tienes { $hours } horas. diff --git a/testing/i18n.toml b/testing/i18n.toml new file mode 100644 index 000000000..dbfa29e5e --- /dev/null +++ b/testing/i18n.toml @@ -0,0 +1,12 @@ +# Defaults to "en": +fallback_language = "en-US" + +[fluent] +# Defaults to true: +use_isolating = false + +# Defaults to "i18n": +assets_dir = "i18n-basic" + +# Default to None: +# core_locales = "…" diff --git a/testing/templates/i18n.html b/testing/templates/i18n.html new file mode 100644 index 000000000..990c1eaeb --- /dev/null +++ b/testing/templates/i18n.html @@ -0,0 +1,2 @@ +

{{ localize("greeting", name: name) }}

+

{{ localize("age", hours: hours ) }}

diff --git a/testing/templates/i18n_no_args.html b/testing/templates/i18n_no_args.html new file mode 100644 index 000000000..e4863bddc --- /dev/null +++ b/testing/templates/i18n_no_args.html @@ -0,0 +1 @@ +

{{ localize("test") }}

diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs new file mode 100644 index 000000000..f1240e3aa --- /dev/null +++ b/testing/tests/i18n.rs @@ -0,0 +1,58 @@ +#![cfg(feature = "i18n")] + +use askama::i18n::{langid, Locale}; +use askama::Template; + +askama::i18n::load!(LOCALES); + +#[derive(Template)] +#[template(path = "i18n.html")] +struct UsesI18n<'a> { + #[locale] + loc: Locale<'a>, + name: &'a str, + hours: f64, +} + +#[derive(Template)] +#[template(path = "i18n_no_args.html")] +struct UsesNoArgsI18n<'a> { + #[locale] + loc: Locale<'a>, +} + +#[test] +fn test_existing_language() { + let template = UsesI18n { + loc: Locale::new(langid!("es-MX"), &LOCALES), + name: "Hilda", + hours: 300072.3, + }; + assert_eq!( + template.render().unwrap(), + r#"

¡Hola, Hilda!

+

Tienes 300072.3 horas.

"# + ) +} + +#[test] +fn test_fallback_language() { + let template = UsesI18n { + loc: Locale::new(langid!("nl-BE"), &LOCALES), + name: "Hilda", + hours: 300072.3, + }; + assert_eq!( + template.render().unwrap(), + r#"

Hello, Hilda!

+

You are 300072.3 hours old.

"# + ) +} + +#[test] +fn test_no_args() { + let template = UsesNoArgsI18n { + loc: Locale::new(langid!("en-US"), &LOCALES), + }; + assert_eq!(template.render().unwrap(), r#"

This is a test

"#) +}