barrett/likewise
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename split methods to tokenize

Browse source
This commit is contained in:
Armin Ronacher 2021-02-02 21:56:55 +01:00
parent b3ae45b118
commit 1f73e01ff1
3 changed files with 46 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/text/inline.rs
View file

@ -17,7 +17,7 @@ impl<'bufs, 's, T: DiffableStr + ?Sized> MultiLookup<'bufs, 's, T> {
let mut seqs = Vec::new();
for (string_idx, string) in strings.iter().enumerate() {
let mut offset = 0;
for word in string.split_unicode_words() {
for word in string.tokenize_unicode_words() {
seqs.push((word, string_idx, offset));
offset += word.len();
}
@ -81,7 +81,7 @@ fn push_values<'s, T: DiffableStr + ?Sized>(
// newlines cause all kinds of wacky stuff if they end up highlighted.
// because of this we want to unemphasize all newlines we encounter.
if emphasized {
for seg in s.split_lines_and_newlines() {
for seg in s.tokenize_lines_and_newlines() {
v[idx].push((!seg.ends_with_newline(), seg));
}
} else {