mirror of https://github.com/boa-dev/boa.git
Browse Source
The new ICU4X release stabilized the `icu_segmenter` component, so this PR implements `Intl.Segmenter` using that as a base. Also, I opted for importing `itertools` instead of copy-pasting the implementation of `TupleWindows` because its design is a lot more complex than `Intersperse`, which we copy-pasted previously. Though, I disabled all `std` features of `itertools` to make it a lot more lightweight, so it shouldn't make much difference in compilation times.pull/2852/head
José Julián Espina
2 years ago
14 changed files with 828 additions and 157 deletions
@ -0,0 +1,152 @@ |
|||||||
|
use boa_gc::{Finalize, Trace}; |
||||||
|
use boa_profiler::Profiler; |
||||||
|
use icu_segmenter::{ |
||||||
|
GraphemeClusterBreakIteratorUtf16, SentenceBreakIteratorUtf16, WordBreakIteratorUtf16, |
||||||
|
}; |
||||||
|
|
||||||
|
use crate::{ |
||||||
|
builtins::{iterable::create_iter_result_object, BuiltInBuilder, IntrinsicObject}, |
||||||
|
context::intrinsics::Intrinsics, |
||||||
|
js_string, |
||||||
|
object::ObjectData, |
||||||
|
property::Attribute, |
||||||
|
realm::Realm, |
||||||
|
Context, JsNativeError, JsObject, JsResult, JsString, JsSymbol, JsValue, |
||||||
|
}; |
||||||
|
|
||||||
|
use super::create_segment_data_object; |
||||||
|
|
||||||
|
pub(crate) enum NativeSegmentIterator<'l, 's> { |
||||||
|
Grapheme(GraphemeClusterBreakIteratorUtf16<'l, 's>), |
||||||
|
Word(WordBreakIteratorUtf16<'l, 's>), |
||||||
|
Sentence(SentenceBreakIteratorUtf16<'l, 's>), |
||||||
|
} |
||||||
|
|
||||||
|
impl Iterator for NativeSegmentIterator<'_, '_> { |
||||||
|
type Item = usize; |
||||||
|
|
||||||
|
fn next(&mut self) -> Option<Self::Item> { |
||||||
|
match self { |
||||||
|
NativeSegmentIterator::Grapheme(g) => g.next(), |
||||||
|
NativeSegmentIterator::Word(w) => w.next(), |
||||||
|
NativeSegmentIterator::Sentence(s) => s.next(), |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
impl NativeSegmentIterator<'_, '_> { |
||||||
|
/// If the iterator is a word break iterator, returns `Some(true)` when the segment preceding
|
||||||
|
/// the current boundary is word-like.
|
||||||
|
pub(crate) fn is_word_like(&self) -> Option<bool> { |
||||||
|
if let Self::Word(w) = self { |
||||||
|
Some(w.is_word_like()) |
||||||
|
} else { |
||||||
|
None |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
#[derive(Debug, Trace, Finalize)] |
||||||
|
pub struct SegmentIterator { |
||||||
|
segmenter: JsObject, |
||||||
|
string: JsString, |
||||||
|
next_segment_index: usize, |
||||||
|
} |
||||||
|
|
||||||
|
impl IntrinsicObject for SegmentIterator { |
||||||
|
fn init(realm: &Realm) { |
||||||
|
let _timer = Profiler::global().start_event("%SegmentIteratorPrototype%", "init"); |
||||||
|
|
||||||
|
BuiltInBuilder::with_intrinsic::<Self>(realm) |
||||||
|
.static_property( |
||||||
|
JsSymbol::to_string_tag(), |
||||||
|
js_string!("Segmenter String Iterator"), |
||||||
|
Attribute::CONFIGURABLE, |
||||||
|
) |
||||||
|
.static_method(Self::next, js_string!("next"), 0) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
|
||||||
|
fn get(intrinsics: &Intrinsics) -> JsObject { |
||||||
|
intrinsics.objects().iterator_prototypes().segment() |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
impl SegmentIterator { |
||||||
|
/// [`CreateSegmentIterator ( segmenter, string )`][spec]
|
||||||
|
///
|
||||||
|
/// [spec]: https://tc39.es/ecma402/#sec-createsegmentiterator
|
||||||
|
pub(crate) fn create( |
||||||
|
segmenter: JsObject, |
||||||
|
string: JsString, |
||||||
|
context: &mut Context<'_>, |
||||||
|
) -> JsObject { |
||||||
|
// 1. Let internalSlotsList be « [[IteratingSegmenter]], [[IteratedString]], [[IteratedStringNextSegmentCodeUnitIndex]] ».
|
||||||
|
// 2. Let iterator be OrdinaryObjectCreate(%SegmentIteratorPrototype%, internalSlotsList).
|
||||||
|
// 3. Set iterator.[[IteratingSegmenter]] to segmenter.
|
||||||
|
// 4. Set iterator.[[IteratedString]] to string.
|
||||||
|
// 5. Set iterator.[[IteratedStringNextSegmentCodeUnitIndex]] to 0.
|
||||||
|
// 6. Return iterator.
|
||||||
|
JsObject::from_proto_and_data( |
||||||
|
context |
||||||
|
.intrinsics() |
||||||
|
.objects() |
||||||
|
.iterator_prototypes() |
||||||
|
.segment(), |
||||||
|
ObjectData::segment_iterator(Self { |
||||||
|
segmenter, |
||||||
|
string, |
||||||
|
next_segment_index: 0, |
||||||
|
}), |
||||||
|
) |
||||||
|
} |
||||||
|
/// [`%SegmentIteratorPrototype%.next ( )`][spec]
|
||||||
|
///
|
||||||
|
/// [spec]: https://tc39.es/ecma402/#sec-%segmentiteratorprototype%.next
|
||||||
|
fn next(this: &JsValue, _: &[JsValue], context: &mut Context<'_>) -> JsResult<JsValue> { |
||||||
|
// 1. Let iterator be the this value.
|
||||||
|
// 2. Perform ? RequireInternalSlot(iterator, [[IteratingSegmenter]]).
|
||||||
|
let mut iter = this.as_object().map(JsObject::borrow_mut).ok_or_else(|| { |
||||||
|
JsNativeError::typ() |
||||||
|
.with_message("`next` can only be called on a `Segment Iterator` object") |
||||||
|
})?; |
||||||
|
let iter = iter.as_segment_iterator_mut().ok_or_else(|| { |
||||||
|
JsNativeError::typ() |
||||||
|
.with_message("`next` can only be called on a `Segment Iterator` object") |
||||||
|
})?; |
||||||
|
|
||||||
|
// 5. Let startIndex be iterator.[[IteratedStringNextSegmentCodeUnitIndex]].
|
||||||
|
let start = iter.next_segment_index; |
||||||
|
|
||||||
|
// 4. Let string be iterator.[[IteratedString]].
|
||||||
|
// 6. Let endIndex be ! FindBoundary(segmenter, string, startIndex, after).
|
||||||
|
let Some((end, is_word_like)) = iter.string.get(start..).and_then(|string| { |
||||||
|
// 3. Let segmenter be iterator.[[IteratingSegmenter]].
|
||||||
|
let segmenter = iter.segmenter.borrow(); |
||||||
|
let segmenter = segmenter |
||||||
|
.as_segmenter() |
||||||
|
.expect("segment iterator object should contain a segmenter"); |
||||||
|
let mut segments = segmenter.native.segment(string); |
||||||
|
// the first elem is always 0.
|
||||||
|
segments.next(); |
||||||
|
segments.next().map(|end| (start + end, segments.is_word_like())) |
||||||
|
}) else { |
||||||
|
// 7. If endIndex is not finite, then
|
||||||
|
// a. Return CreateIterResultObject(undefined, true).
|
||||||
|
return Ok(create_iter_result_object(JsValue::undefined(), true, context)); |
||||||
|
}; |
||||||
|
// 8. Set iterator.[[IteratedStringNextSegmentCodeUnitIndex]] to endIndex.
|
||||||
|
iter.next_segment_index = end; |
||||||
|
|
||||||
|
// 9. Let segmentData be ! CreateSegmentDataObject(segmenter, string, startIndex, endIndex).
|
||||||
|
let segment_data = |
||||||
|
create_segment_data_object(iter.string.clone(), start..end, is_word_like, context); |
||||||
|
|
||||||
|
// 10. Return CreateIterResultObject(segmentData, false).
|
||||||
|
Ok(create_iter_result_object( |
||||||
|
segment_data.into(), |
||||||
|
false, |
||||||
|
context, |
||||||
|
)) |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,142 @@ |
|||||||
|
use boa_gc::{Finalize, Trace}; |
||||||
|
use boa_profiler::Profiler; |
||||||
|
use itertools::Itertools; |
||||||
|
|
||||||
|
use crate::{ |
||||||
|
builtins::{BuiltInBuilder, IntrinsicObject}, |
||||||
|
context::intrinsics::Intrinsics, |
||||||
|
js_string, |
||||||
|
object::ObjectData, |
||||||
|
realm::Realm, |
||||||
|
Context, JsArgs, JsNativeError, JsObject, JsResult, JsString, JsSymbol, JsValue, |
||||||
|
}; |
||||||
|
|
||||||
|
use super::{create_segment_data_object, SegmentIterator}; |
||||||
|
|
||||||
|
#[derive(Debug, Trace, Finalize)] |
||||||
|
pub struct Segments { |
||||||
|
segmenter: JsObject, |
||||||
|
string: JsString, |
||||||
|
} |
||||||
|
|
||||||
|
impl IntrinsicObject for Segments { |
||||||
|
fn init(realm: &Realm) { |
||||||
|
let _timer = Profiler::global().start_event("%SegmentsPrototype%", "init"); |
||||||
|
|
||||||
|
BuiltInBuilder::with_intrinsic::<Self>(realm) |
||||||
|
.static_method(Self::containing, "containing", 1) |
||||||
|
.static_method( |
||||||
|
Self::iterator, |
||||||
|
(JsSymbol::iterator(), js_string!("[Symbol.iterator]")), |
||||||
|
0, |
||||||
|
) |
||||||
|
.build(); |
||||||
|
} |
||||||
|
|
||||||
|
fn get(intrinsics: &Intrinsics) -> JsObject { |
||||||
|
intrinsics.objects().segments_prototype() |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
impl Segments { |
||||||
|
/// [`CreateSegmentsObject ( segmenter, string )`][spec]
|
||||||
|
///
|
||||||
|
/// [spec]: https://tc39.es/ecma402/#sec-createsegmentsobject
|
||||||
|
pub(crate) fn create( |
||||||
|
segmenter: JsObject, |
||||||
|
string: JsString, |
||||||
|
context: &mut Context<'_>, |
||||||
|
) -> JsObject { |
||||||
|
// 1. Let internalSlotsList be « [[SegmentsSegmenter]], [[SegmentsString]] ».
|
||||||
|
// 2. Let segments be OrdinaryObjectCreate(%SegmentsPrototype%, internalSlotsList).
|
||||||
|
// 3. Set segments.[[SegmentsSegmenter]] to segmenter.
|
||||||
|
// 4. Set segments.[[SegmentsString]] to string.
|
||||||
|
// 5. Return segments.
|
||||||
|
JsObject::from_proto_and_data( |
||||||
|
context.intrinsics().objects().segments_prototype(), |
||||||
|
ObjectData::segments(Segments { segmenter, string }), |
||||||
|
) |
||||||
|
} |
||||||
|
|
||||||
|
/// [`%SegmentsPrototype%.containing ( index )`][spec]
|
||||||
|
///
|
||||||
|
/// [spec]: https://tc39.es/ecma402/#sec-%segmentsprototype%.containing
|
||||||
|
fn containing( |
||||||
|
this: &JsValue, |
||||||
|
args: &[JsValue], |
||||||
|
context: &mut Context<'_>, |
||||||
|
) -> JsResult<JsValue> { |
||||||
|
// 1. Let segments be the this value.
|
||||||
|
// 2. Perform ? RequireInternalSlot(segments, [[SegmentsSegmenter]]).
|
||||||
|
let segments = this.as_object().map(JsObject::borrow).ok_or_else(|| { |
||||||
|
JsNativeError::typ() |
||||||
|
.with_message("`containing` can only be called on a `Segments` object") |
||||||
|
})?; |
||||||
|
let segments = segments.as_segments().ok_or_else(|| { |
||||||
|
JsNativeError::typ() |
||||||
|
.with_message("`containing` can only be called on a `Segments` object") |
||||||
|
})?; |
||||||
|
|
||||||
|
// 3. Let segmenter be segments.[[SegmentsSegmenter]].
|
||||||
|
let segmenter = segments.segmenter.borrow(); |
||||||
|
let segmenter = segmenter |
||||||
|
.as_segmenter() |
||||||
|
.expect("segments object should contain a segmenter"); |
||||||
|
|
||||||
|
// 4. Let string be segments.[[SegmentsString]].
|
||||||
|
// 5. Let len be the length of string.
|
||||||
|
let len = segments.string.len() as i64; |
||||||
|
|
||||||
|
// 6. Let n be ? ToIntegerOrInfinity(index).
|
||||||
|
let Some(n) = args |
||||||
|
.get_or_undefined(0) |
||||||
|
.to_integer_or_infinity(context)? |
||||||
|
.as_integer() |
||||||
|
// 7. If n < 0 or n ≥ len, return undefined.
|
||||||
|
.filter(|i| (0..len).contains(i)) |
||||||
|
.map(|n| n as usize) else { |
||||||
|
return Ok(JsValue::undefined()); |
||||||
|
}; |
||||||
|
|
||||||
|
// 8. Let startIndex be ! FindBoundary(segmenter, string, n, before).
|
||||||
|
// 9. Let endIndex be ! FindBoundary(segmenter, string, n, after).
|
||||||
|
let (range, is_word_like) = { |
||||||
|
let mut segments = segmenter.native.segment(&segments.string); |
||||||
|
std::iter::from_fn(|| segments.next().map(|i| (i, segments.is_word_like()))) |
||||||
|
.tuple_windows() |
||||||
|
.find(|((i, _), (j, _))| (*i..*j).contains(&n)) |
||||||
|
.map(|((i, _), (j, word))| ((i..j), word)) |
||||||
|
.expect("string should have at least a length of 1, and `n` must be in range") |
||||||
|
}; |
||||||
|
|
||||||
|
// 10. Return ! CreateSegmentDataObject(segmenter, string, startIndex, endIndex).
|
||||||
|
Ok( |
||||||
|
create_segment_data_object(segments.string.clone(), range, is_word_like, context) |
||||||
|
.into(), |
||||||
|
) |
||||||
|
} |
||||||
|
|
||||||
|
/// [`%SegmentsPrototype% [ @@iterator ] ( )`][spec]
|
||||||
|
///
|
||||||
|
/// [spec]: https://tc39.es/ecma402/#sec-%segmentsprototype%-@@iterator
|
||||||
|
fn iterator(this: &JsValue, _: &[JsValue], context: &mut Context<'_>) -> JsResult<JsValue> { |
||||||
|
// 1. Let segments be the this value.
|
||||||
|
// 2. Perform ? RequireInternalSlot(segments, [[SegmentsSegmenter]]).
|
||||||
|
let segments = this.as_object().map(JsObject::borrow).ok_or_else(|| { |
||||||
|
JsNativeError::typ() |
||||||
|
.with_message("`containing` can only be called on a `Segments` object") |
||||||
|
})?; |
||||||
|
let segments = segments.as_segments().ok_or_else(|| { |
||||||
|
JsNativeError::typ() |
||||||
|
.with_message("`containing` can only be called on a `Segments` object") |
||||||
|
})?; |
||||||
|
|
||||||
|
// 3. Let segmenter be segments.[[SegmentsSegmenter]].
|
||||||
|
// 4. Let string be segments.[[SegmentsString]].
|
||||||
|
// 5. Return ! CreateSegmentIterator(segmenter, string).
|
||||||
|
Ok( |
||||||
|
SegmentIterator::create(segments.segmenter.clone(), segments.string.clone(), context) |
||||||
|
.into(), |
||||||
|
) |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue