diff --git a/src/liballoc/string.rs b/src/liballoc/string.rs index 1d98626e90b..2cb81029f95 100644 --- a/src/liballoc/string.rs +++ b/src/liballoc/string.rs @@ -61,8 +61,8 @@ use core::hash; use core::iter::{FromIterator, FusedIterator}; use core::ops::{self, Add, AddAssign, Index, IndexMut}; use core::ptr; -use core::str as core_str; use core::str::pattern::Pattern; +use std_unicode::lossy; use std_unicode::char::{decode_utf16, REPLACEMENT_CHARACTER}; use borrow::{Cow, ToOwned}; @@ -533,111 +533,34 @@ impl String { /// ``` #[stable(feature = "rust1", since = "1.0.0")] pub fn from_utf8_lossy<'a>(v: &'a [u8]) -> Cow<'a, str> { - let mut i; - match str::from_utf8(v) { - Ok(s) => return Cow::Borrowed(s), - Err(e) => i = e.valid_up_to(), + let mut iter = lossy::Utf8Lossy::from_bytes(v).chunks(); + + let (first_valid, first_broken) = if let Some(chunk) = iter.next() { + let lossy::Utf8LossyChunk { valid, broken } = chunk; + if valid.len() == v.len() { + debug_assert!(broken.is_empty()); + return Cow::Borrowed(valid); + } + (valid, broken) + } else { + return Cow::Borrowed(""); + }; + + const REPLACEMENT: &'static str = "\u{FFFD}"; + + let mut res = String::with_capacity(v.len()); + res.push_str(first_valid); + if !first_broken.is_empty() { + res.push_str(REPLACEMENT); } - const TAG_CONT_U8: u8 = 128; - const REPLACEMENT: &'static [u8] = b"\xEF\xBF\xBD"; // U+FFFD in UTF-8 - let total = v.len(); - fn unsafe_get(xs: &[u8], i: usize) -> u8 { - unsafe { *xs.get_unchecked(i) } - } - fn safe_get(xs: &[u8], i: usize, total: usize) -> u8 { - if i >= total { 0 } else { unsafe_get(xs, i) } - } - - let mut res = String::with_capacity(total); - - if i > 0 { - unsafe { res.as_mut_vec().extend_from_slice(&v[..i]) }; - } - - // subseqidx is the index of the first byte of the subsequence we're - // looking at. It's used to copy a bunch of contiguous good codepoints - // at once instead of copying them one by one. - let mut subseqidx = i; - - while i < total { - let i_ = i; - let byte = unsafe_get(v, i); - i += 1; - - macro_rules! error { () => ({ - unsafe { - if subseqidx != i_ { - res.as_mut_vec().extend_from_slice(&v[subseqidx..i_]); - } - subseqidx = i; - res.as_mut_vec().extend_from_slice(REPLACEMENT); - } - })} - - if byte < 128 { - // subseqidx handles this - } else { - let w = core_str::utf8_char_width(byte); - - match w { - 2 => { - if safe_get(v, i, total) & 192 != TAG_CONT_U8 { - error!(); - continue; - } - i += 1; - } - 3 => { - match (byte, safe_get(v, i, total)) { - (0xE0, 0xA0...0xBF) => (), - (0xE1...0xEC, 0x80...0xBF) => (), - (0xED, 0x80...0x9F) => (), - (0xEE...0xEF, 0x80...0xBF) => (), - _ => { - error!(); - continue; - } - } - i += 1; - if safe_get(v, i, total) & 192 != TAG_CONT_U8 { - error!(); - continue; - } - i += 1; - } - 4 => { - match (byte, safe_get(v, i, total)) { - (0xF0, 0x90...0xBF) => (), - (0xF1...0xF3, 0x80...0xBF) => (), - (0xF4, 0x80...0x8F) => (), - _ => { - error!(); - continue; - } - } - i += 1; - if safe_get(v, i, total) & 192 != TAG_CONT_U8 { - error!(); - continue; - } - i += 1; - if safe_get(v, i, total) & 192 != TAG_CONT_U8 { - error!(); - continue; - } - i += 1; - } - _ => { - error!(); - continue; - } - } + for lossy::Utf8LossyChunk { valid, broken } in iter { + res.push_str(valid); + if !broken.is_empty() { + res.push_str(REPLACEMENT); } } - if subseqidx < total { - unsafe { res.as_mut_vec().extend_from_slice(&v[subseqidx..total]) }; - } + Cow::Owned(res) } diff --git a/src/libstd/ffi/os_str.rs b/src/libstd/ffi/os_str.rs index a5a1b5e5f09..f54d79c201f 100644 --- a/src/libstd/ffi/os_str.rs +++ b/src/libstd/ffi/os_str.rs @@ -9,7 +9,7 @@ // except according to those terms. use borrow::{Borrow, Cow}; -use fmt::{self, Debug}; +use fmt; use mem; use ops; use cmp; @@ -312,8 +312,8 @@ impl Default for OsString { } #[stable(feature = "rust1", since = "1.0.0")] -impl Debug for OsString { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { +impl fmt::Debug for OsString { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { fmt::Debug::fmt(&**self, formatter) } } @@ -669,9 +669,15 @@ impl Hash for OsStr { } #[stable(feature = "rust1", since = "1.0.0")] -impl Debug for OsStr { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { - self.inner.fmt(formatter) +impl fmt::Debug for OsStr { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self.inner, formatter) + } +} + +impl OsStr { + pub(crate) fn display(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.inner, formatter) } } diff --git a/src/libstd/path.rs b/src/libstd/path.rs index e128a4164d7..42a54ed6d75 100644 --- a/src/libstd/path.rs +++ b/src/libstd/path.rs @@ -2281,8 +2281,8 @@ impl AsRef for Path { #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Debug for Path { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { - self.inner.fmt(formatter) + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self.inner, formatter) } } @@ -2314,14 +2314,14 @@ pub struct Display<'a> { #[stable(feature = "rust1", since = "1.0.0")] impl<'a> fmt::Debug for Display<'a> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self.path.to_string_lossy(), f) + fmt::Debug::fmt(&self.path, f) } } #[stable(feature = "rust1", since = "1.0.0")] impl<'a> fmt::Display for Display<'a> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&self.path.to_string_lossy(), f) + self.path.inner.display(f) } } diff --git a/src/libstd/sys/redox/os_str.rs b/src/libstd/sys/redox/os_str.rs index c2bba07f68c..c54286353a9 100644 --- a/src/libstd/sys/redox/os_str.rs +++ b/src/libstd/sys/redox/os_str.rs @@ -12,10 +12,11 @@ /// a `Vec`/`[u8]`. use borrow::Cow; -use fmt::{self, Debug}; +use fmt; use str; use mem; use sys_common::{AsInner, IntoInner}; +use std_unicode::lossy::Utf8Lossy; #[derive(Clone, Hash)] pub struct Buf { @@ -26,15 +27,27 @@ pub struct Slice { pub inner: [u8] } -impl Debug for Slice { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { - self.to_string_lossy().fmt(formatter) +impl fmt::Debug for Slice { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&Utf8Lossy::from_bytes(&self.inner), formatter) } } -impl Debug for Buf { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { - self.as_slice().fmt(formatter) +impl fmt::Display for Slice { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&Utf8Lossy::from_bytes(&self.inner), formatter) + } +} + +impl fmt::Debug for Buf { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self.as_slice(), formatter) + } +} + +impl fmt::Display for Buf { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(self.as_slice(), formatter) } } diff --git a/src/libstd/sys/unix/os_str.rs b/src/libstd/sys/unix/os_str.rs index f5b942d3343..777db17e3e1 100644 --- a/src/libstd/sys/unix/os_str.rs +++ b/src/libstd/sys/unix/os_str.rs @@ -12,10 +12,11 @@ /// a `Vec`/`[u8]`. use borrow::Cow; -use fmt::{self, Debug}; +use fmt; use str; use mem; use sys_common::{AsInner, IntoInner}; +use std_unicode::lossy::Utf8Lossy; #[derive(Clone, Hash)] pub struct Buf { @@ -26,15 +27,27 @@ pub struct Slice { pub inner: [u8] } -impl Debug for Slice { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { - self.to_string_lossy().fmt(formatter) +impl fmt::Debug for Slice { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&Utf8Lossy::from_bytes(&self.inner), formatter) } } -impl Debug for Buf { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { - self.as_slice().fmt(formatter) +impl fmt::Display for Slice { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&Utf8Lossy::from_bytes(&self.inner), formatter) + } +} + +impl fmt::Debug for Buf { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self.as_slice(), formatter) + } +} + +impl fmt::Display for Buf { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(self.as_slice(), formatter) } } diff --git a/src/libstd/sys/windows/os_str.rs b/src/libstd/sys/windows/os_str.rs index f401e7b35c8..3eb4582718b 100644 --- a/src/libstd/sys/windows/os_str.rs +++ b/src/libstd/sys/windows/os_str.rs @@ -12,7 +12,7 @@ /// wrapper around the "WTF-8" encoding; see the `wtf8` module for more. use borrow::Cow; -use fmt::{self, Debug}; +use fmt; use sys_common::wtf8::{Wtf8, Wtf8Buf}; use mem; use sys_common::{AsInner, IntoInner}; @@ -34,9 +34,15 @@ impl AsInner for Buf { } } -impl Debug for Buf { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { - self.as_slice().fmt(formatter) +impl fmt::Debug for Buf { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self.as_slice(), formatter) + } +} + +impl fmt::Display for Buf { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(self.as_slice(), formatter) } } @@ -44,9 +50,15 @@ pub struct Slice { pub inner: Wtf8 } -impl Debug for Slice { - fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { - self.inner.fmt(formatter) +impl fmt::Debug for Slice { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self.inner, formatter) + } +} + +impl fmt::Display for Slice { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.inner, formatter) } } diff --git a/src/libstd/sys_common/wtf8.rs b/src/libstd/sys_common/wtf8.rs index df5e4ef1d88..4e4a6e77d12 100644 --- a/src/libstd/sys_common/wtf8.rs +++ b/src/libstd/sys_common/wtf8.rs @@ -39,7 +39,7 @@ use slice; use str; use sys_common::AsInner; -const UTF8_REPLACEMENT_CHARACTER: &'static [u8] = b"\xEF\xBF\xBD"; +const UTF8_REPLACEMENT_CHARACTER: &'static str = "\u{FFFD}"; /// A Unicode code point: from U+0000 to U+10FFFF. /// @@ -339,7 +339,7 @@ impl Wtf8Buf { Some((surrogate_pos, _)) => { pos = surrogate_pos + 3; self.bytes[surrogate_pos..pos] - .copy_from_slice(UTF8_REPLACEMENT_CHARACTER); + .copy_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); }, None => return unsafe { String::from_utf8_unchecked(self.bytes) } } @@ -438,6 +438,30 @@ impl fmt::Debug for Wtf8 { } } +impl fmt::Display for Wtf8 { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let wtf8_bytes = &self.bytes; + let mut pos = 0; + loop { + match self.next_surrogate(pos) { + Some((surrogate_pos, _)) => { + formatter.write_str(unsafe { + str::from_utf8_unchecked(&wtf8_bytes[pos .. surrogate_pos]) + })?; + formatter.write_str(UTF8_REPLACEMENT_CHARACTER)?; + pos = surrogate_pos + 3; + }, + None => { + formatter.write_str(unsafe { + str::from_utf8_unchecked(&wtf8_bytes[pos..]) + })?; + return Ok(()); + } + } + } + } +} + impl Wtf8 { /// Creates a WTF-8 slice from a UTF-8 `&str` slice. /// @@ -516,13 +540,13 @@ impl Wtf8 { let wtf8_bytes = &self.bytes; let mut utf8_bytes = Vec::with_capacity(self.len()); utf8_bytes.extend_from_slice(&wtf8_bytes[..surrogate_pos]); - utf8_bytes.extend_from_slice(UTF8_REPLACEMENT_CHARACTER); + utf8_bytes.extend_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); let mut pos = surrogate_pos + 3; loop { match self.next_surrogate(pos) { Some((surrogate_pos, _)) => { utf8_bytes.extend_from_slice(&wtf8_bytes[pos .. surrogate_pos]); - utf8_bytes.extend_from_slice(UTF8_REPLACEMENT_CHARACTER); + utf8_bytes.extend_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); pos = surrogate_pos + 3; }, None => { @@ -1200,6 +1224,20 @@ mod tests { assert_eq!(string.to_string_lossy(), expected); } + #[test] + fn wtf8_display() { + fn d(b: &[u8]) -> String { + format!("{}", &unsafe { Wtf8::from_bytes_unchecked(b) }) + } + + assert_eq!("", d("".as_bytes())); + assert_eq!("aé 💩", d("aé 💩".as_bytes())); + + let mut string = Wtf8Buf::from_str("aé 💩"); + string.push(CodePoint::from_u32(0xD800).unwrap()); + assert_eq!("aé 💩�", d(string.as_inner())); + } + #[test] fn wtf8_encode_wide() { let mut string = Wtf8Buf::from_str("aé "); diff --git a/src/libstd_unicode/Cargo.toml b/src/libstd_unicode/Cargo.toml index 28fbd3c1aa9..b3346dbe2fb 100644 --- a/src/libstd_unicode/Cargo.toml +++ b/src/libstd_unicode/Cargo.toml @@ -9,5 +9,9 @@ path = "lib.rs" test = false bench = false +[[test]] +name = "std_unicode_tests" +path = "tests/lib.rs" + [dependencies] core = { path = "../libcore" } diff --git a/src/libstd_unicode/lib.rs b/src/libstd_unicode/lib.rs index 19f2ad1c56f..98624800b4c 100644 --- a/src/libstd_unicode/lib.rs +++ b/src/libstd_unicode/lib.rs @@ -34,6 +34,8 @@ #![feature(char_escape_debug)] #![feature(core_char_ext)] +#![feature(str_internals)] +#![feature(core_intrinsics)] #![feature(decode_utf8)] #![feature(fused)] #![feature(fn_traits)] @@ -45,6 +47,7 @@ mod tables; mod u_str; pub mod char; +pub mod lossy; #[allow(deprecated)] pub mod str { diff --git a/src/libstd_unicode/lossy.rs b/src/libstd_unicode/lossy.rs new file mode 100644 index 00000000000..b914cbcf4b8 --- /dev/null +++ b/src/libstd_unicode/lossy.rs @@ -0,0 +1,198 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use core::str as core_str; +use core::fmt; +use core::fmt::Write; +use char; +use core::intrinsics; + + +/// Lossy UTF-8 string. +#[unstable(feature = "str_internals", issue = "0")] +pub struct Utf8Lossy { + bytes: [u8] +} + +impl Utf8Lossy { + pub fn from_str(s: &str) -> &Utf8Lossy { + Utf8Lossy::from_bytes(s.as_bytes()) + } + + pub fn from_bytes(bytes: &[u8]) -> &Utf8Lossy { + unsafe { intrinsics::transmute(bytes) } + } + + pub fn chunks(&self) -> Utf8LossyChunksIter { + Utf8LossyChunksIter { source: &self.bytes } + } +} + + +/// Iterator over lossy UTF-8 string +#[unstable(feature = "str_internals", issue = "0")] +pub struct Utf8LossyChunksIter<'a> { + source: &'a [u8], +} + +#[unstable(feature = "str_internals", issue = "0")] +#[derive(PartialEq, Eq, Debug)] +pub struct Utf8LossyChunk<'a> { + /// Sequence of valid chars. + /// Can be empty between broken UTF-8 chars. + pub valid: &'a str, + /// Single broken char, empty if none. + /// Empty iff iterator item is last. + pub broken: &'a [u8], +} + +impl<'a> Iterator for Utf8LossyChunksIter<'a> { + type Item = Utf8LossyChunk<'a>; + + fn next(&mut self) -> Option> { + if self.source.len() == 0 { + return None; + } + + const TAG_CONT_U8: u8 = 128; + fn unsafe_get(xs: &[u8], i: usize) -> u8 { + unsafe { *xs.get_unchecked(i) } + } + fn safe_get(xs: &[u8], i: usize) -> u8 { + if i >= xs.len() { 0 } else { unsafe_get(xs, i) } + } + + let mut i = 0; + while i < self.source.len() { + let i_ = i; + + let byte = unsafe_get(self.source, i); + i += 1; + + if byte < 128 { + + } else { + let w = core_str::utf8_char_width(byte); + + macro_rules! error { () => ({ + unsafe { + let r = Utf8LossyChunk { + valid: core_str::from_utf8_unchecked(&self.source[0..i_]), + broken: &self.source[i_..i], + }; + self.source = &self.source[i..]; + return Some(r); + } + })} + + match w { + 2 => { + if safe_get(self.source, i) & 192 != TAG_CONT_U8 { + error!(); + } + i += 1; + } + 3 => { + match (byte, safe_get(self.source, i)) { + (0xE0, 0xA0 ... 0xBF) => (), + (0xE1 ... 0xEC, 0x80 ... 0xBF) => (), + (0xED, 0x80 ... 0x9F) => (), + (0xEE ... 0xEF, 0x80 ... 0xBF) => (), + _ => { + error!(); + } + } + i += 1; + if safe_get(self.source, i) & 192 != TAG_CONT_U8 { + error!(); + } + i += 1; + } + 4 => { + match (byte, safe_get(self.source, i)) { + (0xF0, 0x90 ... 0xBF) => (), + (0xF1 ... 0xF3, 0x80 ... 0xBF) => (), + (0xF4, 0x80 ... 0x8F) => (), + _ => { + error!(); + } + } + i += 1; + if safe_get(self.source, i) & 192 != TAG_CONT_U8 { + error!(); + } + i += 1; + if safe_get(self.source, i) & 192 != TAG_CONT_U8 { + error!(); + } + i += 1; + } + _ => { + error!(); + } + } + } + } + + let r = Utf8LossyChunk { + valid: unsafe { core_str::from_utf8_unchecked(self.source) }, + broken: &[], + }; + self.source = &[]; + return Some(r); + } +} + + +impl fmt::Display for Utf8Lossy { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for Utf8LossyChunk { valid, broken } in self.chunks() { + f.write_str(valid)?; + if !broken.is_empty() { + f.write_char(char::REPLACEMENT_CHARACTER)?; + } + } + Ok(()) + } +} + +impl fmt::Debug for Utf8Lossy { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_char('"')?; + + for Utf8LossyChunk { valid, broken } in self.chunks() { + + // Valid part. + // Here we partially parse UTF-8 again which is suboptimal. + { + let mut from = 0; + for (i, c) in valid.char_indices() { + let esc = c.escape_debug(); + // If char needs escaping, flush backlog so far and write, else skip + if esc.len() != 1 { + f.write_str(&valid[from..i])?; + for c in esc { + f.write_char(c)?; + } + from = i + c.len_utf8(); + } + } + f.write_str(&valid[from..])?; + } + + // Broken parts of string as hex escape. + for &b in broken { + write!(f, "\\x{:02x}", b)?; + } + } + + f.write_char('"') + } +} diff --git a/src/libstd_unicode/tests/lib.rs b/src/libstd_unicode/tests/lib.rs new file mode 100644 index 00000000000..9535ec18763 --- /dev/null +++ b/src/libstd_unicode/tests/lib.rs @@ -0,0 +1,15 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(str_internals, unicode)] + +extern crate std_unicode; + +mod lossy; diff --git a/src/libstd_unicode/tests/lossy.rs b/src/libstd_unicode/tests/lossy.rs new file mode 100644 index 00000000000..e05d0668556 --- /dev/null +++ b/src/libstd_unicode/tests/lossy.rs @@ -0,0 +1,91 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std_unicode::lossy::*; + +#[test] +fn chunks() { + let mut iter = Utf8Lossy::from_bytes(b"hello").chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "hello", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); + + let mut iter = Utf8Lossy::from_bytes("ศไทย中华Việt Nam".as_bytes()).chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "ศไทย中华Việt Nam", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); + + let mut iter = Utf8Lossy::from_bytes(b"Hello\xC2 There\xFF Goodbye").chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "Hello", broken: b"\xC2", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: " There", broken: b"\xFF", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: " Goodbye", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); + + let mut iter = Utf8Lossy::from_bytes(b"Hello\xC0\x80 There\xE6\x83 Goodbye").chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "Hello", broken: b"\xC0", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: " There", broken: b"\xE6\x83", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: " Goodbye", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); + + let mut iter = Utf8Lossy::from_bytes(b"\xF5foo\xF5\x80bar").chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xF5", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "foo", broken: b"\xF5", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "bar", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); + + let mut iter = Utf8Lossy::from_bytes(b"\xF1foo\xF1\x80bar\xF1\x80\x80baz").chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xF1", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "foo", broken: b"\xF1\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "bar", broken: b"\xF1\x80\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "baz", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); + + let mut iter = Utf8Lossy::from_bytes(b"\xF4foo\xF4\x80bar\xF4\xBFbaz").chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xF4", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "foo", broken: b"\xF4\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "bar", broken: b"\xF4", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xBF", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "baz", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); + + let mut iter = Utf8Lossy::from_bytes(b"\xF0\x80\x80\x80foo\xF0\x90\x80\x80bar").chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xF0", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "foo\u{10000}bar", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); + + // surrogates + let mut iter = Utf8Lossy::from_bytes(b"\xED\xA0\x80foo\xED\xBF\xBFbar").chunks(); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xED", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xA0", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\x80", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "foo", broken: b"\xED", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xBF", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "", broken: b"\xBF", }), iter.next()); + assert_eq!(Some(Utf8LossyChunk { valid: "bar", broken: b"", }), iter.next()); + assert_eq!(None, iter.next()); +} + +#[test] +fn display() { + assert_eq!( + "Hello\u{FFFD}\u{FFFD} There\u{FFFD} Goodbye", + &format!("{}", Utf8Lossy::from_bytes(b"Hello\xC0\x80 There\xE6\x83 Goodbye"))); +} + +#[test] +fn debug() { + assert_eq!( + "\"Hello\\xc0\\x80 There\\xe6\\x83 Goodbye\\u{10d4ea}\"", + &format!("{:?}", Utf8Lossy::from_bytes( + b"Hello\xC0\x80 There\xE6\x83 Goodbye\xf4\x8d\x93\xaa"))); +}