switch to upstream unescape
This commit is contained in:
parent
c79eea9fc1
commit
2473cb6a5c
6
Cargo.lock
generated
6
Cargo.lock
generated
|
@ -1310,7 +1310,7 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ra_rustc_lexer"
|
name = "ra_rustc_lexer"
|
||||||
version = "0.1.0-pre.1"
|
version = "0.1.0-pre.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
@ -1322,7 +1322,7 @@ version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"ra_parser 0.1.0",
|
"ra_parser 0.1.0",
|
||||||
"ra_rustc_lexer 0.1.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
"ra_rustc_lexer 0.1.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"ra_text_edit 0.1.0",
|
"ra_text_edit 0.1.0",
|
||||||
"rowan 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
"rowan 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"smol_str 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
"smol_str 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
|
@ -2259,7 +2259,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
"checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
|
"checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
|
||||||
"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
|
"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
|
||||||
"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
||||||
"checksum ra_rustc_lexer 0.1.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e8d92772f822978a6c9c4657aa61af439e4e635180628b3354049b283b749f1e"
|
"checksum ra_rustc_lexer 0.1.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6baccda91574dfadd7f8a0bc8f9f110f874b6b484289b2536d3dbf4f0d5d97bb"
|
||||||
"checksum ra_vfs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fb7cd4e302032c5ab514f1c01c89727cd96fd950dd36f9ebee9252df45d9fb1a"
|
"checksum ra_vfs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fb7cd4e302032c5ab514f1c01c89727cd96fd950dd36f9ebee9252df45d9fb1a"
|
||||||
"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
|
"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
|
||||||
"checksum rand 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d47eab0e83d9693d40f825f86948aa16eff6750ead4bdffc4ab95b8b3a7f052c"
|
"checksum rand 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d47eab0e83d9693d40f825f86948aa16eff6750ead4bdffc4ab95b8b3a7f052c"
|
||||||
|
|
|
@ -11,7 +11,7 @@ repository = "https://github.com/rust-analyzer/rust-analyzer"
|
||||||
unicode-xid = "0.1.0"
|
unicode-xid = "0.1.0"
|
||||||
itertools = "0.8.0"
|
itertools = "0.8.0"
|
||||||
rowan = "0.6.0"
|
rowan = "0.6.0"
|
||||||
ra_rustc_lexer = { version = "0.1.0-pre.1", features = [ "unicode-xid" ] }
|
ra_rustc_lexer = { version = "0.1.0-pre.2" }
|
||||||
|
|
||||||
# ideally, `serde` should be enabled by `ra_lsp_server`, but we enable it here
|
# ideally, `serde` should be enabled by `ra_lsp_server`, but we enable it here
|
||||||
# to reduce number of compilations
|
# to reduce number of compilations
|
||||||
|
|
|
@ -1,16 +1,99 @@
|
||||||
mod unescape;
|
|
||||||
|
|
||||||
mod block;
|
mod block;
|
||||||
mod field_expr;
|
mod field_expr;
|
||||||
|
|
||||||
|
use ra_rustc_lexer::unescape;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
algo::visit::{visitor_ctx, VisitorCtx},
|
algo::visit::{visitor_ctx, VisitorCtx},
|
||||||
ast, SyntaxError,
|
ast, SyntaxError, SyntaxErrorKind,
|
||||||
SyntaxKind::{BYTE, BYTE_STRING, CHAR, STRING},
|
SyntaxKind::{BYTE, BYTE_STRING, CHAR, STRING},
|
||||||
SyntaxNode, TextUnit, T,
|
SyntaxNode, TextUnit, T,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) use unescape::EscapeError;
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||||
|
pub enum EscapeError {
|
||||||
|
ZeroChars,
|
||||||
|
MoreThanOneChar,
|
||||||
|
LoneSlash,
|
||||||
|
InvalidEscape,
|
||||||
|
BareCarriageReturn,
|
||||||
|
EscapeOnlyChar,
|
||||||
|
TooShortHexEscape,
|
||||||
|
InvalidCharInHexEscape,
|
||||||
|
OutOfRangeHexEscape,
|
||||||
|
NoBraceInUnicodeEscape,
|
||||||
|
InvalidCharInUnicodeEscape,
|
||||||
|
EmptyUnicodeEscape,
|
||||||
|
UnclosedUnicodeEscape,
|
||||||
|
LeadingUnderscoreUnicodeEscape,
|
||||||
|
OverlongUnicodeEscape,
|
||||||
|
LoneSurrogateUnicodeEscape,
|
||||||
|
OutOfRangeUnicodeEscape,
|
||||||
|
UnicodeEscapeInByte,
|
||||||
|
NonAsciiCharInByte,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ra_rustc_lexer::unescape::EscapeError> for EscapeError {
|
||||||
|
fn from(err: ra_rustc_lexer::unescape::EscapeError) -> Self {
|
||||||
|
match err {
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::ZeroChars => EscapeError::ZeroChars,
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::MoreThanOneChar => EscapeError::MoreThanOneChar,
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::LoneSlash => EscapeError::LoneSlash,
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::InvalidEscape => EscapeError::InvalidEscape,
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::BareCarriageReturn
|
||||||
|
| ra_rustc_lexer::unescape::EscapeError::BareCarriageReturnInRawString => {
|
||||||
|
EscapeError::BareCarriageReturn
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::EscapeOnlyChar => EscapeError::EscapeOnlyChar,
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::TooShortHexEscape => {
|
||||||
|
EscapeError::TooShortHexEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::InvalidCharInHexEscape => {
|
||||||
|
EscapeError::InvalidCharInHexEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::OutOfRangeHexEscape => {
|
||||||
|
EscapeError::OutOfRangeHexEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::NoBraceInUnicodeEscape => {
|
||||||
|
EscapeError::NoBraceInUnicodeEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::InvalidCharInUnicodeEscape => {
|
||||||
|
EscapeError::InvalidCharInUnicodeEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::EmptyUnicodeEscape => {
|
||||||
|
EscapeError::EmptyUnicodeEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::UnclosedUnicodeEscape => {
|
||||||
|
EscapeError::UnclosedUnicodeEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::LeadingUnderscoreUnicodeEscape => {
|
||||||
|
EscapeError::LeadingUnderscoreUnicodeEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::OverlongUnicodeEscape => {
|
||||||
|
EscapeError::OverlongUnicodeEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::LoneSurrogateUnicodeEscape => {
|
||||||
|
EscapeError::LoneSurrogateUnicodeEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::OutOfRangeUnicodeEscape => {
|
||||||
|
EscapeError::OutOfRangeUnicodeEscape
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::UnicodeEscapeInByte => {
|
||||||
|
EscapeError::UnicodeEscapeInByte
|
||||||
|
}
|
||||||
|
ra_rustc_lexer::unescape::EscapeError::NonAsciiCharInByte
|
||||||
|
| ra_rustc_lexer::unescape::EscapeError::NonAsciiCharInByteString => {
|
||||||
|
EscapeError::NonAsciiCharInByte
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ra_rustc_lexer::unescape::EscapeError> for SyntaxErrorKind {
|
||||||
|
fn from(err: ra_rustc_lexer::unescape::EscapeError) -> Self {
|
||||||
|
SyntaxErrorKind::EscapeError(err.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn validate(root: &SyntaxNode) -> Vec<SyntaxError> {
|
pub(crate) fn validate(root: &SyntaxNode) -> Vec<SyntaxError> {
|
||||||
let mut errors = Vec::new();
|
let mut errors = Vec::new();
|
||||||
|
|
|
@ -1,521 +0,0 @@
|
||||||
//! Utilities for validating string and char literals and turning them into
|
|
||||||
//! values they represent.
|
|
||||||
//!
|
|
||||||
//! This file is copy-pasted from the compiler
|
|
||||||
//!
|
|
||||||
//! https://github.com/rust-lang/rust/blob/c6ac57564852cb6e2d0db60f7b46d9eb98d4b449/src/libsyntax/parse/unescape.rs
|
|
||||||
//!
|
|
||||||
//! Hopefully, we'll share this code in a proper way some day
|
|
||||||
|
|
||||||
use std::ops::Range;
|
|
||||||
use std::str::Chars;
|
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq, Clone, Hash)]
|
|
||||||
pub enum EscapeError {
|
|
||||||
ZeroChars,
|
|
||||||
MoreThanOneChar,
|
|
||||||
|
|
||||||
LoneSlash,
|
|
||||||
InvalidEscape,
|
|
||||||
BareCarriageReturn,
|
|
||||||
EscapeOnlyChar,
|
|
||||||
|
|
||||||
TooShortHexEscape,
|
|
||||||
InvalidCharInHexEscape,
|
|
||||||
OutOfRangeHexEscape,
|
|
||||||
|
|
||||||
NoBraceInUnicodeEscape,
|
|
||||||
InvalidCharInUnicodeEscape,
|
|
||||||
EmptyUnicodeEscape,
|
|
||||||
UnclosedUnicodeEscape,
|
|
||||||
LeadingUnderscoreUnicodeEscape,
|
|
||||||
OverlongUnicodeEscape,
|
|
||||||
LoneSurrogateUnicodeEscape,
|
|
||||||
OutOfRangeUnicodeEscape,
|
|
||||||
|
|
||||||
UnicodeEscapeInByte,
|
|
||||||
NonAsciiCharInByte,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Takes a contents of a char literal (without quotes), and returns an
|
|
||||||
/// unescaped char or an error
|
|
||||||
pub(crate) fn unescape_char(literal_text: &str) -> Result<char, (usize, EscapeError)> {
|
|
||||||
let mut chars = literal_text.chars();
|
|
||||||
unescape_char_or_byte(&mut chars, Mode::Char)
|
|
||||||
.map_err(|err| (literal_text.len() - chars.as_str().len(), err))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Takes a contents of a string literal (without quotes) and produces a
|
|
||||||
/// sequence of escaped characters or errors.
|
|
||||||
pub(crate) fn unescape_str<F>(literal_text: &str, callback: &mut F)
|
|
||||||
where
|
|
||||||
F: FnMut(Range<usize>, Result<char, EscapeError>),
|
|
||||||
{
|
|
||||||
unescape_str_or_byte_str(literal_text, Mode::Str, callback)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn unescape_byte(literal_text: &str) -> Result<u8, (usize, EscapeError)> {
|
|
||||||
let mut chars = literal_text.chars();
|
|
||||||
unescape_char_or_byte(&mut chars, Mode::Byte)
|
|
||||||
.map(byte_from_char)
|
|
||||||
.map_err(|err| (literal_text.len() - chars.as_str().len(), err))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Takes a contents of a string literal (without quotes) and produces a
|
|
||||||
/// sequence of escaped characters or errors.
|
|
||||||
pub(crate) fn unescape_byte_str<F>(literal_text: &str, callback: &mut F)
|
|
||||||
where
|
|
||||||
F: FnMut(Range<usize>, Result<u8, EscapeError>),
|
|
||||||
{
|
|
||||||
unescape_str_or_byte_str(literal_text, Mode::ByteStr, &mut |range, char| {
|
|
||||||
callback(range, char.map(byte_from_char))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
|
||||||
pub(crate) enum Mode {
|
|
||||||
Char,
|
|
||||||
Str,
|
|
||||||
Byte,
|
|
||||||
ByteStr,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Mode {
|
|
||||||
fn in_single_quotes(self) -> bool {
|
|
||||||
match self {
|
|
||||||
Mode::Char | Mode::Byte => true,
|
|
||||||
Mode::Str | Mode::ByteStr => false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn in_double_quotes(self) -> bool {
|
|
||||||
!self.in_single_quotes()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn is_bytes(self) -> bool {
|
|
||||||
match self {
|
|
||||||
Mode::Byte | Mode::ByteStr => true,
|
|
||||||
Mode::Char | Mode::Str => false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn scan_escape(first_char: char, chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
|
|
||||||
if first_char != '\\' {
|
|
||||||
return match first_char {
|
|
||||||
'\t' | '\n' => Err(EscapeError::EscapeOnlyChar),
|
|
||||||
'\r' => Err(if chars.clone().next() == Some('\n') {
|
|
||||||
EscapeError::EscapeOnlyChar
|
|
||||||
} else {
|
|
||||||
EscapeError::BareCarriageReturn
|
|
||||||
}),
|
|
||||||
'\'' if mode.in_single_quotes() => Err(EscapeError::EscapeOnlyChar),
|
|
||||||
'"' if mode.in_double_quotes() => Err(EscapeError::EscapeOnlyChar),
|
|
||||||
_ => {
|
|
||||||
if mode.is_bytes() && !first_char.is_ascii() {
|
|
||||||
return Err(EscapeError::NonAsciiCharInByte);
|
|
||||||
}
|
|
||||||
Ok(first_char)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
let second_char = chars.next().ok_or(EscapeError::LoneSlash)?;
|
|
||||||
|
|
||||||
let res = match second_char {
|
|
||||||
'"' => '"',
|
|
||||||
'n' => '\n',
|
|
||||||
'r' => '\r',
|
|
||||||
't' => '\t',
|
|
||||||
'\\' => '\\',
|
|
||||||
'\'' => '\'',
|
|
||||||
'0' => '\0',
|
|
||||||
|
|
||||||
'x' => {
|
|
||||||
let hi = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
|
|
||||||
let hi = hi.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
|
|
||||||
|
|
||||||
let lo = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
|
|
||||||
let lo = lo.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
|
|
||||||
|
|
||||||
let value = hi * 16 + lo;
|
|
||||||
|
|
||||||
if !mode.is_bytes() && !is_ascii(value) {
|
|
||||||
return Err(EscapeError::OutOfRangeHexEscape);
|
|
||||||
}
|
|
||||||
let value = value as u8;
|
|
||||||
|
|
||||||
value as char
|
|
||||||
}
|
|
||||||
|
|
||||||
'u' => {
|
|
||||||
if chars.next() != Some('{') {
|
|
||||||
return Err(EscapeError::NoBraceInUnicodeEscape);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut n_digits = 1;
|
|
||||||
let mut value: u32 = match chars.next().ok_or(EscapeError::UnclosedUnicodeEscape)? {
|
|
||||||
'_' => return Err(EscapeError::LeadingUnderscoreUnicodeEscape),
|
|
||||||
'}' => return Err(EscapeError::EmptyUnicodeEscape),
|
|
||||||
c => c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?,
|
|
||||||
};
|
|
||||||
|
|
||||||
loop {
|
|
||||||
match chars.next() {
|
|
||||||
None => return Err(EscapeError::UnclosedUnicodeEscape),
|
|
||||||
Some('_') => continue,
|
|
||||||
Some('}') => {
|
|
||||||
if n_digits > 6 {
|
|
||||||
return Err(EscapeError::OverlongUnicodeEscape);
|
|
||||||
}
|
|
||||||
if mode.is_bytes() {
|
|
||||||
return Err(EscapeError::UnicodeEscapeInByte);
|
|
||||||
}
|
|
||||||
|
|
||||||
break std::char::from_u32(value).ok_or_else(|| {
|
|
||||||
if value > 0x0010_FFFF {
|
|
||||||
EscapeError::OutOfRangeUnicodeEscape
|
|
||||||
} else {
|
|
||||||
EscapeError::LoneSurrogateUnicodeEscape
|
|
||||||
}
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
Some(c) => {
|
|
||||||
let digit =
|
|
||||||
c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?;
|
|
||||||
n_digits += 1;
|
|
||||||
if n_digits > 6 {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let digit = digit as u32;
|
|
||||||
value = value * 16 + digit;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => return Err(EscapeError::InvalidEscape),
|
|
||||||
};
|
|
||||||
Ok(res)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
|
|
||||||
let first_char = chars.next().ok_or(EscapeError::ZeroChars)?;
|
|
||||||
let res = scan_escape(first_char, chars, mode)?;
|
|
||||||
if chars.next().is_some() {
|
|
||||||
return Err(EscapeError::MoreThanOneChar);
|
|
||||||
}
|
|
||||||
Ok(res)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Takes a contents of a string literal (without quotes) and produces a
|
|
||||||
/// sequence of escaped characters or errors.
|
|
||||||
fn unescape_str_or_byte_str<F>(src: &str, mode: Mode, callback: &mut F)
|
|
||||||
where
|
|
||||||
F: FnMut(Range<usize>, Result<char, EscapeError>),
|
|
||||||
{
|
|
||||||
assert!(mode.in_double_quotes());
|
|
||||||
let initial_len = src.len();
|
|
||||||
let mut chars = src.chars();
|
|
||||||
while let Some(first_char) = chars.next() {
|
|
||||||
let start = initial_len - chars.as_str().len() - first_char.len_utf8();
|
|
||||||
|
|
||||||
let unescaped_char = match first_char {
|
|
||||||
'\\' => {
|
|
||||||
let (second_char, third_char) = {
|
|
||||||
let mut chars = chars.clone();
|
|
||||||
(chars.next(), chars.next())
|
|
||||||
};
|
|
||||||
match (second_char, third_char) {
|
|
||||||
(Some('\n'), _) | (Some('\r'), Some('\n')) => {
|
|
||||||
skip_ascii_whitespace(&mut chars);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
_ => scan_escape(first_char, &mut chars, mode),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
'\r' => {
|
|
||||||
let second_char = chars.clone().next();
|
|
||||||
if second_char == Some('\n') {
|
|
||||||
chars.next();
|
|
||||||
Ok('\n')
|
|
||||||
} else {
|
|
||||||
scan_escape(first_char, &mut chars, mode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
'\n' => Ok('\n'),
|
|
||||||
'\t' => Ok('\t'),
|
|
||||||
_ => scan_escape(first_char, &mut chars, mode),
|
|
||||||
};
|
|
||||||
let end = initial_len - chars.as_str().len();
|
|
||||||
callback(start..end, unescaped_char);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn skip_ascii_whitespace(chars: &mut Chars<'_>) {
|
|
||||||
let str = chars.as_str();
|
|
||||||
let first_non_space = str
|
|
||||||
.bytes()
|
|
||||||
.position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r')
|
|
||||||
.unwrap_or_else(|| str.len());
|
|
||||||
*chars = str[first_non_space..].chars()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn byte_from_char(c: char) -> u8 {
|
|
||||||
let res = c as u32;
|
|
||||||
assert!(res <= u32::from(u8::max_value()), "guaranteed because of Mode::Byte");
|
|
||||||
res as u8
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_ascii(x: u32) -> bool {
|
|
||||||
x <= 0x7F
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_unescape_char_bad() {
|
|
||||||
fn check(literal_text: &str, expected_error: EscapeError) {
|
|
||||||
let actual_result = unescape_char(literal_text).map_err(|(_offset, err)| err);
|
|
||||||
assert_eq!(actual_result, Err(expected_error));
|
|
||||||
}
|
|
||||||
|
|
||||||
check("", EscapeError::ZeroChars);
|
|
||||||
check(r"\", EscapeError::LoneSlash);
|
|
||||||
|
|
||||||
check("\n", EscapeError::EscapeOnlyChar);
|
|
||||||
check("\r\n", EscapeError::EscapeOnlyChar);
|
|
||||||
check("\t", EscapeError::EscapeOnlyChar);
|
|
||||||
check("'", EscapeError::EscapeOnlyChar);
|
|
||||||
check("\r", EscapeError::BareCarriageReturn);
|
|
||||||
|
|
||||||
check("spam", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\x0ff", EscapeError::MoreThanOneChar);
|
|
||||||
check(r#"\"a"#, EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\na", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\ra", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\ta", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\\a", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\'a", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\0a", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\u{0}x", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\u{1F63b}}", EscapeError::MoreThanOneChar);
|
|
||||||
|
|
||||||
check(r"\v", EscapeError::InvalidEscape);
|
|
||||||
check(r"\💩", EscapeError::InvalidEscape);
|
|
||||||
check(r"\●", EscapeError::InvalidEscape);
|
|
||||||
|
|
||||||
check(r"\x", EscapeError::TooShortHexEscape);
|
|
||||||
check(r"\x0", EscapeError::TooShortHexEscape);
|
|
||||||
check(r"\xf", EscapeError::TooShortHexEscape);
|
|
||||||
check(r"\xa", EscapeError::TooShortHexEscape);
|
|
||||||
check(r"\xx", EscapeError::InvalidCharInHexEscape);
|
|
||||||
check(r"\xы", EscapeError::InvalidCharInHexEscape);
|
|
||||||
check(r"\x🦀", EscapeError::InvalidCharInHexEscape);
|
|
||||||
check(r"\xtt", EscapeError::InvalidCharInHexEscape);
|
|
||||||
check(r"\xff", EscapeError::OutOfRangeHexEscape);
|
|
||||||
check(r"\xFF", EscapeError::OutOfRangeHexEscape);
|
|
||||||
check(r"\x80", EscapeError::OutOfRangeHexEscape);
|
|
||||||
|
|
||||||
check(r"\u", EscapeError::NoBraceInUnicodeEscape);
|
|
||||||
check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape);
|
|
||||||
check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape);
|
|
||||||
check(r"\u{", EscapeError::UnclosedUnicodeEscape);
|
|
||||||
check(r"\u{0000", EscapeError::UnclosedUnicodeEscape);
|
|
||||||
check(r"\u{}", EscapeError::EmptyUnicodeEscape);
|
|
||||||
check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape);
|
|
||||||
check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape);
|
|
||||||
check(r"\u{FFFFFF}", EscapeError::OutOfRangeUnicodeEscape);
|
|
||||||
check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape);
|
|
||||||
check(r"\u{ffffff}", EscapeError::OutOfRangeUnicodeEscape);
|
|
||||||
|
|
||||||
check(r"\u{DC00}", EscapeError::LoneSurrogateUnicodeEscape);
|
|
||||||
check(r"\u{DDDD}", EscapeError::LoneSurrogateUnicodeEscape);
|
|
||||||
check(r"\u{DFFF}", EscapeError::LoneSurrogateUnicodeEscape);
|
|
||||||
|
|
||||||
check(r"\u{D800}", EscapeError::LoneSurrogateUnicodeEscape);
|
|
||||||
check(r"\u{DAAA}", EscapeError::LoneSurrogateUnicodeEscape);
|
|
||||||
check(r"\u{DBFF}", EscapeError::LoneSurrogateUnicodeEscape);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_unescape_char_good() {
|
|
||||||
fn check(literal_text: &str, expected_char: char) {
|
|
||||||
let actual_result = unescape_char(literal_text);
|
|
||||||
assert_eq!(actual_result, Ok(expected_char));
|
|
||||||
}
|
|
||||||
|
|
||||||
check("a", 'a');
|
|
||||||
check("ы", 'ы');
|
|
||||||
check("🦀", '🦀');
|
|
||||||
|
|
||||||
check(r#"\""#, '"');
|
|
||||||
check(r"\n", '\n');
|
|
||||||
check(r"\r", '\r');
|
|
||||||
check(r"\t", '\t');
|
|
||||||
check(r"\\", '\\');
|
|
||||||
check(r"\'", '\'');
|
|
||||||
check(r"\0", '\0');
|
|
||||||
|
|
||||||
check(r"\x00", '\0');
|
|
||||||
check(r"\x5a", 'Z');
|
|
||||||
check(r"\x5A", 'Z');
|
|
||||||
check(r"\x7f", 127 as char);
|
|
||||||
|
|
||||||
check(r"\u{0}", '\0');
|
|
||||||
check(r"\u{000000}", '\0');
|
|
||||||
check(r"\u{41}", 'A');
|
|
||||||
check(r"\u{0041}", 'A');
|
|
||||||
check(r"\u{00_41}", 'A');
|
|
||||||
check(r"\u{4__1__}", 'A');
|
|
||||||
check(r"\u{1F63b}", '😻');
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_unescape_str_good() {
|
|
||||||
fn check(literal_text: &str, expected: &str) {
|
|
||||||
let mut buf = Ok(String::with_capacity(literal_text.len()));
|
|
||||||
unescape_str(literal_text, &mut |range, c| {
|
|
||||||
if let Ok(b) = &mut buf {
|
|
||||||
match c {
|
|
||||||
Ok(c) => b.push(c),
|
|
||||||
Err(e) => buf = Err((range, e)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let buf = buf.as_ref().map(|it| it.as_ref());
|
|
||||||
assert_eq!(buf, Ok(expected))
|
|
||||||
}
|
|
||||||
|
|
||||||
check("foo", "foo");
|
|
||||||
check("", "");
|
|
||||||
check(" \t\n\r\n", " \t\n\n");
|
|
||||||
|
|
||||||
check("hello \\\n world", "hello world");
|
|
||||||
check("hello \\\r\n world", "hello world");
|
|
||||||
check("thread's", "thread's")
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_unescape_byte_bad() {
|
|
||||||
fn check(literal_text: &str, expected_error: EscapeError) {
|
|
||||||
let actual_result = unescape_byte(literal_text).map_err(|(_offset, err)| err);
|
|
||||||
assert_eq!(actual_result, Err(expected_error));
|
|
||||||
}
|
|
||||||
|
|
||||||
check("", EscapeError::ZeroChars);
|
|
||||||
check(r"\", EscapeError::LoneSlash);
|
|
||||||
|
|
||||||
check("\n", EscapeError::EscapeOnlyChar);
|
|
||||||
check("\r\n", EscapeError::EscapeOnlyChar);
|
|
||||||
check("\t", EscapeError::EscapeOnlyChar);
|
|
||||||
check("'", EscapeError::EscapeOnlyChar);
|
|
||||||
check("\r", EscapeError::BareCarriageReturn);
|
|
||||||
|
|
||||||
check("spam", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\x0ff", EscapeError::MoreThanOneChar);
|
|
||||||
check(r#"\"a"#, EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\na", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\ra", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\ta", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\\a", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\'a", EscapeError::MoreThanOneChar);
|
|
||||||
check(r"\0a", EscapeError::MoreThanOneChar);
|
|
||||||
|
|
||||||
check(r"\v", EscapeError::InvalidEscape);
|
|
||||||
check(r"\💩", EscapeError::InvalidEscape);
|
|
||||||
check(r"\●", EscapeError::InvalidEscape);
|
|
||||||
|
|
||||||
check(r"\x", EscapeError::TooShortHexEscape);
|
|
||||||
check(r"\x0", EscapeError::TooShortHexEscape);
|
|
||||||
check(r"\xa", EscapeError::TooShortHexEscape);
|
|
||||||
check(r"\xf", EscapeError::TooShortHexEscape);
|
|
||||||
check(r"\xx", EscapeError::InvalidCharInHexEscape);
|
|
||||||
check(r"\xы", EscapeError::InvalidCharInHexEscape);
|
|
||||||
check(r"\x🦀", EscapeError::InvalidCharInHexEscape);
|
|
||||||
check(r"\xtt", EscapeError::InvalidCharInHexEscape);
|
|
||||||
|
|
||||||
check(r"\u", EscapeError::NoBraceInUnicodeEscape);
|
|
||||||
check(r"\u[0123]", EscapeError::NoBraceInUnicodeEscape);
|
|
||||||
check(r"\u{0x}", EscapeError::InvalidCharInUnicodeEscape);
|
|
||||||
check(r"\u{", EscapeError::UnclosedUnicodeEscape);
|
|
||||||
check(r"\u{0000", EscapeError::UnclosedUnicodeEscape);
|
|
||||||
check(r"\u{}", EscapeError::EmptyUnicodeEscape);
|
|
||||||
check(r"\u{_0000}", EscapeError::LeadingUnderscoreUnicodeEscape);
|
|
||||||
check(r"\u{0000000}", EscapeError::OverlongUnicodeEscape);
|
|
||||||
|
|
||||||
check("ы", EscapeError::NonAsciiCharInByte);
|
|
||||||
check("🦀", EscapeError::NonAsciiCharInByte);
|
|
||||||
|
|
||||||
check(r"\u{0}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{000000}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{41}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{0041}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{00_41}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{4__1__}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{1F63b}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{0}x", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{1F63b}}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{FFFFFF}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{ffffff}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{DC00}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{DDDD}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{DFFF}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{D800}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{DAAA}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
check(r"\u{DBFF}", EscapeError::UnicodeEscapeInByte);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_unescape_byte_good() {
|
|
||||||
fn check(literal_text: &str, expected_byte: u8) {
|
|
||||||
let actual_result = unescape_byte(literal_text);
|
|
||||||
assert_eq!(actual_result, Ok(expected_byte));
|
|
||||||
}
|
|
||||||
|
|
||||||
check("a", b'a');
|
|
||||||
|
|
||||||
check(r#"\""#, b'"');
|
|
||||||
check(r"\n", b'\n');
|
|
||||||
check(r"\r", b'\r');
|
|
||||||
check(r"\t", b'\t');
|
|
||||||
check(r"\\", b'\\');
|
|
||||||
check(r"\'", b'\'');
|
|
||||||
check(r"\0", b'\0');
|
|
||||||
|
|
||||||
check(r"\x00", b'\0');
|
|
||||||
check(r"\x5a", b'Z');
|
|
||||||
check(r"\x5A", b'Z');
|
|
||||||
check(r"\x7f", 127);
|
|
||||||
check(r"\x80", 128);
|
|
||||||
check(r"\xff", 255);
|
|
||||||
check(r"\xFF", 255);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_unescape_byte_str_good() {
|
|
||||||
fn check(literal_text: &str, expected: &[u8]) {
|
|
||||||
let mut buf = Ok(Vec::with_capacity(literal_text.len()));
|
|
||||||
unescape_byte_str(literal_text, &mut |range, c| {
|
|
||||||
if let Ok(b) = &mut buf {
|
|
||||||
match c {
|
|
||||||
Ok(c) => b.push(c),
|
|
||||||
Err(e) => buf = Err((range, e)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let buf = buf.as_ref().map(|it| it.as_ref());
|
|
||||||
assert_eq!(buf, Ok(expected))
|
|
||||||
}
|
|
||||||
|
|
||||||
check("foo", b"foo");
|
|
||||||
check("", b"");
|
|
||||||
check(" \t\n\r\n", b" \t\n\n");
|
|
||||||
|
|
||||||
check("hello \\\n world", b"hello world");
|
|
||||||
check("hello \\\r\n world", b"hello world");
|
|
||||||
check("thread's", b"thread's")
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in a new issue