Skip to content

Commit 107ae47

Browse files
committed
tokenizer: Source positions are always at char boundaries, so avoid overhead when slicing.
These show up in profiles: https://share.firefox.dev/45Gh2s7
1 parent a5ba6dd commit 107ae47

File tree

1 file changed

+7
-4
lines changed

1 file changed

+7
-4
lines changed

src/tokenizer.rs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ impl<'a> Tokenizer<'a> {
274274

275275
#[inline]
276276
pub fn position(&self) -> SourcePosition {
277+
debug_assert!(self.input.is_char_boundary(self.position));
277278
SourcePosition(self.position)
278279
}
279280

@@ -313,13 +314,15 @@ impl<'a> Tokenizer<'a> {
313314
}
314315

315316
#[inline]
316-
pub fn slice_from(&self, start_pos: SourcePosition) -> &'a str {
317-
&self.input[start_pos.0..self.position]
317+
pub(crate) fn slice_from(&self, start_pos: SourcePosition) -> &'a str {
318+
self.slice(start_pos..self.position())
318319
}
319320

320321
#[inline]
321-
pub fn slice(&self, range: Range<SourcePosition>) -> &'a str {
322-
&self.input[range.start.0..range.end.0]
322+
pub(crate) fn slice(&self, range: Range<SourcePosition>) -> &'a str {
323+
debug_assert!(self.input.is_char_boundary(range.start.0));
324+
debug_assert!(self.input.is_char_boundary(range.end.0));
325+
unsafe { self.input.get_unchecked(range.start.0..range.end.0) }
323326
}
324327

325328
pub fn current_source_line(&self) -> &'a str {

0 commit comments

Comments
 (0)