mirror of https://github.com/boa-dev/boa.git
Browse Source
* Split default icu data into lazily deserialized parts * FIx no_std compilation * Lazily load more ICU tools * Fix regressions and use more stable constructorspull/3798/merge
José Julián Espina
3 months ago
committed by
GitHub
25 changed files with 385 additions and 207 deletions
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -1,86 +1,80 @@ |
|||||||
#![allow(missing_docs, rustdoc::missing_crate_level_docs)] |
#![allow(missing_docs, rustdoc::missing_crate_level_docs)] |
||||||
|
|
||||||
use std::{error::Error, fs::File, path::Path}; |
use std::path::Path; |
||||||
|
use std::{error::Error, fs::File}; |
||||||
|
|
||||||
use icu_datagen::blob_exporter::BlobExporter; |
use icu_datagen::blob_exporter::BlobExporter; |
||||||
use icu_datagen::prelude::*; |
use icu_datagen::prelude::*; |
||||||
use icu_provider::data_key; |
|
||||||
|
|
||||||
const KEYS_LEN: usize = 129; |
/// Path to the directory where the exported data lives.
|
||||||
|
const EXPORT_PATH: &str = "core/icu_provider/data"; |
||||||
|
|
||||||
/// List of keys used by `Intl` components.
|
/// List of services used by `Intl` components.
|
||||||
///
|
///
|
||||||
/// This must be kept in sync with the list of implemented components of `Intl`.
|
/// This must be kept in sync with the list of implemented services of `Intl`.
|
||||||
const KEYS: [DataKey; KEYS_LEN] = { |
const SERVICES: &[(&str, &[DataKey])] = &[ |
||||||
const CENTINEL_KEY: DataKey = data_key!("centinel@1"); |
("icu_casemap", icu_casemap::provider::KEYS), |
||||||
const SERVICES: [&[DataKey]; 9] = [ |
("icu_collator", icu_collator::provider::KEYS), |
||||||
icu_casemap::provider::KEYS, |
("icu_datetime", icu_datetime::provider::KEYS), |
||||||
icu_collator::provider::KEYS, |
("icu_decimal", icu_decimal::provider::KEYS), |
||||||
icu_datetime::provider::KEYS, |
("icu_list", icu_list::provider::KEYS), |
||||||
icu_decimal::provider::KEYS, |
("icu_locid_transform", icu_locid_transform::provider::KEYS), |
||||||
icu_list::provider::KEYS, |
("icu_normalizer", icu_normalizer::provider::KEYS), |
||||||
icu_locid_transform::provider::KEYS, |
("icu_plurals", icu_plurals::provider::KEYS), |
||||||
icu_normalizer::provider::KEYS, |
("icu_segmenter", icu_segmenter::provider::KEYS), |
||||||
icu_plurals::provider::KEYS, |
|
||||||
icu_segmenter::provider::KEYS, |
|
||||||
]; |
]; |
||||||
|
|
||||||
let mut array = [CENTINEL_KEY; KEYS_LEN]; |
fn export_for_service( |
||||||
|
service: &str, |
||||||
let mut offset = 0; |
keys: &[DataKey], |
||||||
let mut service_idx = 0; |
provider: &DatagenProvider, |
||||||
|
driver: DatagenDriver, |
||||||
while service_idx < SERVICES.len() { |
) -> Result<(), Box<dyn Error>> { |
||||||
let service = SERVICES[service_idx]; |
log::info!( |
||||||
let mut idx = 0; |
"Generating ICU4X data for service `{service}` with keys: {:#?}", |
||||||
while idx < service.len() { |
keys |
||||||
array[offset + idx] = service[idx]; |
); |
||||||
idx += 1; |
|
||||||
} |
let export_path = Path::new(EXPORT_PATH); |
||||||
|
let export_file = export_path.join(format!("{service}.postcard")); |
||||||
|
|
||||||
|
driver.with_keys(keys.iter().copied()).export( |
||||||
|
provider, |
||||||
|
BlobExporter::new_v2_with_sink(Box::new(File::create(export_file)?)), |
||||||
|
)?; |
||||||
|
|
||||||
offset += service.len(); |
Ok(()) |
||||||
service_idx += 1; |
|
||||||
} |
} |
||||||
|
|
||||||
assert!(offset == array.len()); |
|
||||||
|
|
||||||
array |
|
||||||
}; |
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn Error>> { |
fn main() -> Result<(), Box<dyn Error>> { |
||||||
simple_logger::SimpleLogger::new() |
simple_logger::SimpleLogger::new() |
||||||
.env() |
.env() |
||||||
.with_level(log::LevelFilter::Info) |
.with_level(log::LevelFilter::Info) |
||||||
.init()?; |
.init()?; |
||||||
|
|
||||||
let path = Path::new("core/icu_provider/data"); |
|
||||||
|
|
||||||
// Removal will throw an error if the directory doesn't exist, hence
|
// Removal will throw an error if the directory doesn't exist, hence
|
||||||
// why we can ignore the error.
|
// why we can ignore the error.
|
||||||
let _unused = std::fs::remove_dir_all(path); |
let _unused = std::fs::remove_dir_all(EXPORT_PATH); |
||||||
std::fs::create_dir_all(path)?; |
std::fs::create_dir_all(EXPORT_PATH)?; |
||||||
|
|
||||||
log::info!("Generating ICU4X data for keys: {:#?}", KEYS); |
|
||||||
|
|
||||||
let provider = DatagenProvider::new_latest_tested(); |
let provider = &DatagenProvider::new_latest_tested(); |
||||||
let locales = provider |
let locales = provider |
||||||
.locales_for_coverage_levels([CoverageLevel::Modern])? |
.locales_for_coverage_levels([CoverageLevel::Modern])? |
||||||
.into_iter() |
.into_iter() |
||||||
.chain([langid!("en-US")]); |
.chain([langid!("en-US")]); |
||||||
|
|
||||||
DatagenDriver::new() |
let driver = DatagenDriver::new() |
||||||
.with_keys(KEYS) |
|
||||||
.with_locales_and_fallback(locales.map(LocaleFamily::with_descendants), { |
.with_locales_and_fallback(locales.map(LocaleFamily::with_descendants), { |
||||||
let mut options = FallbackOptions::default(); |
let mut options = FallbackOptions::default(); |
||||||
options.deduplication_strategy = Some(DeduplicationStrategy::None); |
options.deduplication_strategy = Some(DeduplicationStrategy::None); |
||||||
options |
options |
||||||
}) |
}) |
||||||
.with_additional_collations([String::from("search*")]) |
.with_additional_collations([String::from("search*")]) |
||||||
.with_recommended_segmenter_models() |
.with_recommended_segmenter_models(); |
||||||
.export( |
|
||||||
&provider, |
for (service, keys) in SERVICES { |
||||||
BlobExporter::new_v2_with_sink(Box::new(File::create(path.join("icudata.postcard"))?)), |
export_for_service(service, keys, provider, driver.clone())?; |
||||||
)?; |
} |
||||||
|
|
||||||
Ok(()) |
Ok(()) |
||||||
} |
} |
||||||
|
|||||||
Loading…
Reference in new issue