bstr/src/utf8.rs

use core::char;
use core::cmp;
use core::fmt;
use core::str;
#[cfg(feature = "std")]
use std::error;

use crate::ascii;
use crate::bstr::BStr;
use crate::ext_slice::ByteSlice;

// The UTF-8 decoder provided here is based on the one presented here:
// https://bjoern.hoehrmann.de/utf-8/decoder/dfa/
//
// We *could* have done UTF-8 decoding by using a DFA generated by `\p{any}`
// using regex-automata that is roughly the same size. The real benefit of
// Hoehrmann's formulation is that the byte class mapping below is manually
// tailored such that each byte's class doubles as a shift to mask out the
// bits necessary for constructing the leading bits of each codepoint value
// from the initial byte.
//
// There are some minor differences between this implementation and Hoehrmann's
// formulation.
//
// Firstly, we make REJECT have state ID 0, since it makes the state table
// itself a little easier to read and is consistent with the notion that 0
// means "false" or "bad."
//
// Secondly, when doing bulk decoding, we add a SIMD accelerated ASCII fast
// path.
//
// Thirdly, we pre-multiply the state IDs to avoid a multiplication instruction
// in the core decoding loop. (Which is what regex-automata would do by
// default.)
//
// Fourthly, we split the byte class mapping and transition table into two
// arrays because it's clearer.
//
// It is unlikely that this is the fastest way to do UTF-8 decoding, however,
// it is fairly simple.

const ACCEPT: usize = 12;
const REJECT: usize = 0;

/// SAFETY: The decode below function relies on the correctness of these
/// equivalence classes.
#[cfg_attr(rustfmt, rustfmt::skip)]
const CLASSES: [u8; 256] = [
   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
   7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
   8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
  10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
];

/// SAFETY: The decode below function relies on the correctness of this state
/// machine.
#[cfg_attr(rustfmt, rustfmt::skip)]
const STATES_FORWARD: &'static [u8] = &[
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  12, 0, 24, 36, 60, 96, 84, 0, 0, 0, 48, 72,
  0, 12, 0, 0, 0, 0, 0, 12, 0, 12, 0, 0,
  0, 24, 0, 0, 0, 0, 0, 24, 0, 24, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0,
  0, 24, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 36, 0, 36, 0, 0,
  0, 36, 0, 0, 0, 0, 0, 36, 0, 36, 0, 0,
  0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
];

/// An iterator over Unicode scalar values in a byte string.
///
/// When invalid UTF-8 byte sequences are found, they are substituted with the
/// Unicode replacement codepoint (`U+FFFD`) using the
/// ["maximal subpart" strategy](http://www.unicode.org/review/pr-121.html).
///
/// This iterator is created by the
/// [`chars`](trait.ByteSlice.html#method.chars) method provided by the
/// [`ByteSlice`](trait.ByteSlice.html) extension trait for `&[u8]`.
#[derive(Clone, Debug)]
pub struct Chars<'a> {
    bs: &'a [u8],
}

impl<'a> Chars<'a> {
    pub(crate) fn new(bs: &'a [u8]) -> Chars<'a> {
        Chars { bs }
    }

    /// View the underlying data as a subslice of the original data.
    ///
    /// The slice returned has the same lifetime as the original slice, and so
    /// the iterator can continue to be used while this exists.
    ///
    /// # Examples
    ///
    /// ```
    /// use bstr::ByteSlice;
    ///
    /// let mut chars = b"abc".chars();
    ///
    /// assert_eq!(b"abc", chars.as_bytes());
    /// chars.next();
    /// assert_eq!(b"bc", chars.as_bytes());
    /// chars.next();
    /// chars.next();
    /// assert_eq!(b"", chars.as_bytes());
    /// ```
    #[inline]
    pub fn as_bytes(&self) -> &'a [u8] {
        self.bs
    }
}

impl<'a> Iterator for Chars<'a> {
    type Item = char;

    #[inline]
    fn next(&mut self) -> Option<char> {
        let (ch, size) = decode_lossy(self.bs);
        if size == 0 {
            return None;
        }
        self.bs = &self.bs[size..];
        Some(ch)
    }
}

impl<'a> DoubleEndedIterator for Chars<'a> {
    #[inline]
    fn next_back(&mut self) -> Option<char> {
        let (ch, size) = decode_last_lossy(self.bs);
        if size == 0 {
            return None;
        }
        self.bs = &self.bs[..self.bs.len() - size];
        Some(ch)
    }
}

/// An iterator over Unicode scalar values in a byte string and their
/// byte index positions.
///
/// When invalid UTF-8 byte sequences are found, they are substituted with the
/// Unicode replacement codepoint (`U+FFFD`) using the
/// ["maximal subpart" strategy](http://www.unicode.org/review/pr-121.html).
///
/// Note that this is slightly different from the `CharIndices` iterator
/// provided by the standard library. Aside from working on possibly invalid
/// UTF-8, this iterator provides both the corresponding starting and ending
/// byte indices of each codepoint yielded. The ending position is necessary to
/// slice the original byte string when invalid UTF-8 bytes are converted into
/// a Unicode replacement codepoint, since a single replacement codepoint can
/// substitute anywhere from 1 to 3 invalid bytes (inclusive).
///
/// This iterator is created by the
/// [`char_indices`](trait.ByteSlice.html#method.char_indices) method provided
/// by the [`ByteSlice`](trait.ByteSlice.html) extension trait for `&[u8]`.
#[derive(Clone, Debug)]
pub struct CharIndices<'a> {
    bs: &'a [u8],
    forward_index: usize,
    reverse_index: usize,
}

impl<'a> CharIndices<'a> {
    pub(crate) fn new(bs: &'a [u8]) -> CharIndices<'a> {
        CharIndices { bs: bs, forward_index: 0, reverse_index: bs.len() }
    }

    /// View the underlying data as a subslice of the original data.
    ///
    /// The slice returned has the same lifetime as the original slice, and so
    /// the iterator can continue to be used while this exists.
    ///
    /// # Examples
    ///
    /// ```
    /// use bstr::ByteSlice;
    ///
    /// let mut it = b"abc".char_indices();
    ///
    /// assert_eq!(b"abc", it.as_bytes());
    /// it.next();
    /// assert_eq!(b"bc", it.as_bytes());
    /// it.next();
    /// it.next();
    /// assert_eq!(b"", it.as_bytes());
    /// ```
    #[inline]
    pub fn as_bytes(&self) -> &'a [u8] {
        self.bs
    }
}

impl<'a> Iterator for CharIndices<'a> {
    type Item = (usize, usize, char);

    #[inline]
    fn next(&mut self) -> Option<(usize, usize, char)> {
        let index = self.forward_index;
        let (ch, size) = decode_lossy(self.bs);
        if size == 0 {
            return None;
        }
        self.bs = &self.bs[size..];
        self.forward_index += size;
        Some((index, index + size, ch))
    }
}

impl<'a> DoubleEndedIterator for CharIndices<'a> {
    #[inline]
    fn next_back(&mut self) -> Option<(usize, usize, char)> {
        let (ch, size) = decode_last_lossy(self.bs);
        if size == 0 {
            return None;
        }
        self.bs = &self.bs[..self.bs.len() - size];
        self.reverse_index -= size;
        Some((self.reverse_index, self.reverse_index + size, ch))
    }
}

impl<'a> ::core::iter::FusedIterator for CharIndices<'a> {}

/// An iterator over chunks of valid UTF-8 in a byte slice.
///
/// See [`utf8_chunks`](trait.ByteSlice.html#method.utf8_chunks).
#[derive(Clone, Debug)]
pub struct Utf8Chunks<'a> {
    pub(super) bytes: &'a [u8],
}

/// A chunk of valid UTF-8, possibly followed by invalid UTF-8 bytes.
///
/// This is yielded by the
/// [`Utf8Chunks`](struct.Utf8Chunks.html)
/// iterator, which can be created via the
/// [`ByteSlice::utf8_chunks`](trait.ByteSlice.html#method.utf8_chunks)
/// method.
///
/// The `'a` lifetime parameter corresponds to the lifetime of the bytes that
/// are being iterated over.
#[cfg_attr(test, derive(Debug, PartialEq))]
pub struct Utf8Chunk<'a> {
    /// A valid UTF-8 piece, at the start, end, or between invalid UTF-8 bytes.
    ///
    /// This is empty between adjacent invalid UTF-8 byte sequences.
    valid: &'a str,
    /// A sequence of invalid UTF-8 bytes.
    ///
    /// Can only be empty in the last chunk.
    ///
    /// Should be replaced by a single unicode replacement character, if not
    /// empty.
    invalid: &'a BStr,
    /// Indicates whether the invalid sequence could've been valid if there
    /// were more bytes.
    ///
    /// Can only be true in the last chunk.
    incomplete: bool,
}

impl<'a> Utf8Chunk<'a> {
    /// Returns the (possibly empty) valid UTF-8 bytes in this chunk.
    ///
    /// This may be empty if there are consecutive sequences of invalid UTF-8
    /// bytes.
    #[inline]
    pub fn valid(&self) -> &'a str {
        self.valid
    }

    /// Returns the (possibly empty) invalid UTF-8 bytes in this chunk that
    /// immediately follow the valid UTF-8 bytes in this chunk.
    ///
    /// This is only empty when this chunk corresponds to the last chunk in
    /// the original bytes.
    ///
    /// The maximum length of this slice is 3. That is, invalid UTF-8 byte
    /// sequences greater than 1 always correspond to a valid _prefix_ of
    /// a valid UTF-8 encoded codepoint. This corresponds to the "substitution
    /// of maximal subparts" strategy that is described in more detail in the
    /// docs for the
    /// [`ByteSlice::to_str_lossy`](trait.ByteSlice.html#method.to_str_lossy)
    /// method.
    #[inline]
    pub fn invalid(&self) -> &'a [u8] {
        self.invalid.as_bytes()
    }

    /// Returns whether the invalid sequence might still become valid if more
    /// bytes are added.
    ///
    /// Returns true if the end of the input was reached unexpectedly,
    /// without encountering an unexpected byte.
    ///
    /// This can only be the case for the last chunk.
    #[inline]
    pub fn incomplete(&self) -> bool {
        self.incomplete
    }
}

impl<'a> Iterator for Utf8Chunks<'a> {
    type Item = Utf8Chunk<'a>;

    #[inline]
    fn next(&mut self) -> Option<Utf8Chunk<'a>> {
        if self.bytes.is_empty() {
            return None;
        }
        match validate(self.bytes) {
            Ok(()) => {
                let valid = self.bytes;
                self.bytes = &[];
                Some(Utf8Chunk {
                    // SAFETY: This is safe because of the guarantees provided
                    // by utf8::validate.
                    valid: unsafe { str::from_utf8_unchecked(valid) },
                    invalid: [].as_bstr(),
                    incomplete: false,
                })
            }
            Err(e) => {
                let (valid, rest) = self.bytes.split_at(e.valid_up_to());
                // SAFETY: This is safe because of the guarantees provided by
                // utf8::validate.
                let valid = unsafe { str::from_utf8_unchecked(valid) };
                let (invalid_len, incomplete) = match e.error_len() {
                    Some(n) => (n, false),
                    None => (rest.len(), true),
                };
                let (invalid, rest) = rest.split_at(invalid_len);
                self.bytes = rest;
                Some(Utf8Chunk {
                    valid,
                    invalid: invalid.as_bstr(),
                    incomplete,
                })
            }
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        if self.bytes.is_empty() {
            (0, Some(0))
        } else {
            (1, Some(self.bytes.len()))
        }
    }
}

impl<'a> ::core::iter::FusedIterator for Utf8Chunks<'a> {}

/// An error that occurs when UTF-8 decoding fails.
///
/// This error occurs when attempting to convert a non-UTF-8 byte
/// string to a Rust string that must be valid UTF-8. For example,
/// [`to_str`](trait.ByteSlice.html#method.to_str) is one such method.
///
/// # Example
///
/// This example shows what happens when a given byte sequence is invalid,
/// but ends with a sequence that is a possible prefix of valid UTF-8.
///
/// ```
/// use bstr::{B, ByteSlice};
///
/// let s = B(b"foobar\xF1\x80\x80");
/// let err = s.to_str().unwrap_err();
/// assert_eq!(err.valid_up_to(), 6);
/// assert_eq!(err.error_len(), None);
/// ```
///
/// This example shows what happens when a given byte sequence contains
/// invalid UTF-8.
///
/// ```
/// use bstr::ByteSlice;
///
/// let s = b"foobar\xF1\x80\x80quux";
/// let err = s.to_str().unwrap_err();
/// assert_eq!(err.valid_up_to(), 6);
/// // The error length reports the maximum number of bytes that correspond to
/// // a valid prefix of a UTF-8 encoded codepoint.
/// assert_eq!(err.error_len(), Some(3));
///
/// // In contrast to the above which contains a single invalid prefix,
/// // consider the case of multiple individal bytes that are never valid
/// // prefixes. Note how the value of error_len changes!
/// let s = b"foobar\xFF\xFFquux";
/// let err = s.to_str().unwrap_err();
/// assert_eq!(err.valid_up_to(), 6);
/// assert_eq!(err.error_len(), Some(1));
///
/// // The fact that it's an invalid prefix does not change error_len even
/// // when it immediately precedes the end of the string.
/// let s = b"foobar\xFF";
/// let err = s.to_str().unwrap_err();
/// assert_eq!(err.valid_up_to(), 6);
/// assert_eq!(err.error_len(), Some(1));
/// ```
#[derive(Debug, Eq, PartialEq)]
pub struct Utf8Error {
    valid_up_to: usize,
    error_len: Option<usize>,
}

impl Utf8Error {
    /// Returns the byte index of the position immediately following the last
    /// valid UTF-8 byte.
    ///
    /// # Example
    ///
    /// This examples shows how `valid_up_to` can be used to retrieve a
    /// possibly empty prefix that is guaranteed to be valid UTF-8:
    ///
    /// ```
    /// use bstr::ByteSlice;
    ///
    /// let s = b"foobar\xF1\x80\x80quux";
    /// let err = s.to_str().unwrap_err();
    ///
    /// // This is guaranteed to never panic.
    /// let string = s[..err.valid_up_to()].to_str().unwrap();
    /// assert_eq!(string, "foobar");
    /// ```
    #[inline]
    pub fn valid_up_to(&self) -> usize {
        self.valid_up_to
    }

    /// Returns the total number of invalid UTF-8 bytes immediately following
    /// the position returned by `valid_up_to`. This value is always at least
    /// `1`, but can be up to `3` if bytes form a valid prefix of some UTF-8
    /// encoded codepoint.
    ///
    /// If the end of the original input was found before a valid UTF-8 encoded
    /// codepoint could be completed, then this returns `None`. This is useful
    /// when processing streams, where a `None` value signals that more input
    /// might be needed.
    #[inline]
    pub fn error_len(&self) -> Option<usize> {
        self.error_len
    }
}

#[cfg(feature = "std")]
impl error::Error for Utf8Error {
    fn description(&self) -> &str {
        "invalid UTF-8"
    }
}

impl fmt::Display for Utf8Error {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "invalid UTF-8 found at byte offset {}", self.valid_up_to)
    }
}

/// Returns OK if and only if the given slice is completely valid UTF-8.
///
/// If the slice isn't valid UTF-8, then an error is returned that explains
/// the first location at which invalid UTF-8 was detected.
pub fn validate(slice: &[u8]) -> Result<(), Utf8Error> {
    // The fast path for validating UTF-8. It steps through a UTF-8 automaton
    // and uses a SIMD accelerated ASCII fast path on x86_64. If an error is
    // detected, it backs up and runs the slower version of the UTF-8 automaton
    // to determine correct error information.
    fn fast(slice: &[u8]) -> Result<(), Utf8Error> {
        let mut state = ACCEPT;
        let mut i = 0;

        while i < slice.len() {
            let b = slice[i];

            // ASCII fast path. If we see two consecutive ASCII bytes, then try
            // to validate as much ASCII as possible very quickly.
            if state == ACCEPT
                && b <= 0x7F
                && slice.get(i + 1).map_or(false, |&b| b <= 0x7F)
            {
                i += ascii::first_non_ascii_byte(&slice[i..]);
                continue;
            }

            state = step(state, b);
            if state == REJECT {
                return Err(find_valid_up_to(slice, i));
            }
            i += 1;
        }
        if state != ACCEPT {
            Err(find_valid_up_to(slice, slice.len()))
        } else {
            Ok(())
        }
    }

    // Given the first position at which a UTF-8 sequence was determined to be
    // invalid, return an error that correctly reports the position at which
    // the last complete UTF-8 sequence ends.
    #[inline(never)]
    fn find_valid_up_to(slice: &[u8], rejected_at: usize) -> Utf8Error {
        // In order to find the last valid byte, we need to back up an amount
        // that guarantees every preceding byte is part of a valid UTF-8
        // code unit sequence. To do this, we simply locate the last leading
        // byte that occurs before rejected_at.
        let mut backup = rejected_at.saturating_sub(1);
        while backup > 0 && !is_leading_or_invalid_utf8_byte(slice[backup]) {
            backup -= 1;
        }
        let upto = cmp::min(slice.len(), rejected_at.saturating_add(1));
        let mut err = slow(&slice[backup..upto]).unwrap_err();
        err.valid_up_to += backup;
        err
    }

    // Like top-level UTF-8 decoding, except it correctly reports a UTF-8 error
    // when an invalid sequence is found. This is split out from validate so
    // that the fast path doesn't need to keep track of the position of the
    // last valid UTF-8 byte. In particular, tracking this requires checking
    // for an ACCEPT state on each byte, which degrades throughput pretty
    // badly.
    fn slow(slice: &[u8]) -> Result<(), Utf8Error> {
        let mut state = ACCEPT;
        let mut valid_up_to = 0;
        for (i, &b) in slice.iter().enumerate() {
            state = step(state, b);
            if state == ACCEPT {
                valid_up_to = i + 1;
            } else if state == REJECT {
                // Our error length must always be at least 1.
                let error_len = Some(cmp::max(1, i - valid_up_to));
                return Err(Utf8Error { valid_up_to, error_len });
            }
        }
        if state != ACCEPT {
            Err(Utf8Error { valid_up_to, error_len: None })
        } else {
            Ok(())
        }
    }

    // Advance to the next state given the current state and current byte.
    fn step(state: usize, b: u8) -> usize {
        let class = CLASSES[b as usize];
        // SAFETY: This is safe because 'class' is always <=11 and 'state' is
        // always <=96. Therefore, the maximal index is 96+11 = 107, where
        // STATES_FORWARD.len() = 108 such that every index is guaranteed to be
        // valid by construction of the state machine and the byte equivalence
        // classes.
        unsafe {
            *STATES_FORWARD.get_unchecked(state + class as usize) as usize
        }
    }

    fast(slice)
}

/// UTF-8 decode a single Unicode scalar value from the beginning of a slice.
///
/// When successful, the corresponding Unicode scalar value is returned along
/// with the number of bytes it was encoded with. The number of bytes consumed
/// for a successful decode is always between 1 and 4, inclusive.
///
/// When unsuccessful, `None` is returned along with the number of bytes that
/// make up a maximal prefix of a valid UTF-8 code unit sequence. In this case,
/// the number of bytes consumed is always between 0 and 3, inclusive, where
/// 0 is only returned when `slice` is empty.
///
/// # Examples
///
/// Basic usage:
///
/// ```
/// use bstr::decode_utf8;
///
/// // Decoding a valid codepoint.
/// let (ch, size) = decode_utf8(b"\xE2\x98\x83");
/// assert_eq!(Some('☃'), ch);
/// assert_eq!(3, size);
///
/// // Decoding an incomplete codepoint.
/// let (ch, size) = decode_utf8(b"\xE2\x98");
/// assert_eq!(None, ch);
/// assert_eq!(2, size);
/// ```
///
/// This example shows how to iterate over all codepoints in UTF-8 encoded
/// bytes, while replacing invalid UTF-8 sequences with the replacement
/// codepoint:
///
/// ```
/// use bstr::{B, decode_utf8};
///
/// let mut bytes = B(b"\xE2\x98\x83\xFF\xF0\x9D\x9E\x83\xE2\x98\x61");
/// let mut chars = vec![];
/// while !bytes.is_empty() {
///     let (ch, size) = decode_utf8(bytes);
///     bytes = &bytes[size..];
///     chars.push(ch.unwrap_or('\u{FFFD}'));
/// }
/// assert_eq!(vec!['☃', '\u{FFFD}', '��', '\u{FFFD}', 'a'], chars);
/// ```
#[inline]
pub fn decode<B: AsRef<[u8]>>(slice: B) -> (Option<char>, usize) {
    let slice = slice.as_ref();
    match slice.get(0) {
        None => return (None, 0),
        Some(&b) if b <= 0x7F => return (Some(b as char), 1),
        _ => {}
    }

    let (mut state, mut cp, mut i) = (ACCEPT, 0, 0);
    while i < slice.len() {
        decode_step(&mut state, &mut cp, slice[i]);
        i += 1;

        if state == ACCEPT {
            // SAFETY: This is safe because `decode_step` guarantees that
            // `cp` is a valid Unicode scalar value in an ACCEPT state.
            let ch = unsafe { char::from_u32_unchecked(cp) };
            return (Some(ch), i);
        } else if state == REJECT {
            // At this point, we always want to advance at least one byte.
            return (None, cmp::max(1, i.saturating_sub(1)));
        }
    }
    (None, i)
}

/// Lossily UTF-8 decode a single Unicode scalar value from the beginning of a
/// slice.
///
/// When successful, the corresponding Unicode scalar value is returned along
/// with the number of bytes it was encoded with. The number of bytes consumed
/// for a successful decode is always between 1 and 4, inclusive.
///
/// When unsuccessful, the Unicode replacement codepoint (`U+FFFD`) is returned
/// along with the number of bytes that make up a maximal prefix of a valid
/// UTF-8 code unit sequence. In this case, the number of bytes consumed is
/// always between 0 and 3, inclusive, where 0 is only returned when `slice` is
/// empty.
///
/// # Examples
///
/// Basic usage:
///
/// ```ignore
/// use bstr::decode_utf8_lossy;
///
/// // Decoding a valid codepoint.
/// let (ch, size) = decode_utf8_lossy(b"\xE2\x98\x83");
/// assert_eq!('☃', ch);
/// assert_eq!(3, size);
///
/// // Decoding an incomplete codepoint.
/// let (ch, size) = decode_utf8_lossy(b"\xE2\x98");
/// assert_eq!('\u{FFFD}', ch);
/// assert_eq!(2, size);
/// ```
///
/// This example shows how to iterate over all codepoints in UTF-8 encoded
/// bytes, while replacing invalid UTF-8 sequences with the replacement
/// codepoint:
///
/// ```ignore
/// use bstr::{B, decode_utf8_lossy};
///
/// let mut bytes = B(b"\xE2\x98\x83\xFF\xF0\x9D\x9E\x83\xE2\x98\x61");
/// let mut chars = vec![];
/// while !bytes.is_empty() {
///     let (ch, size) = decode_utf8_lossy(bytes);
///     bytes = &bytes[size..];
///     chars.push(ch);
/// }
/// assert_eq!(vec!['☃', '\u{FFFD}', '��', '\u{FFFD}', 'a'], chars);
/// ```
#[inline]
pub fn decode_lossy<B: AsRef<[u8]>>(slice: B) -> (char, usize) {
    match decode(slice) {
        (Some(ch), size) => (ch, size),
        (None, size) => ('\u{FFFD}', size),
    }
}

/// UTF-8 decode a single Unicode scalar value from the end of a slice.
///
/// When successful, the corresponding Unicode scalar value is returned along
/// with the number of bytes it was encoded with. The number of bytes consumed
/// for a successful decode is always between 1 and 4, inclusive.
///
/// When unsuccessful, `None` is returned along with the number of bytes that
/// make up a maximal prefix of a valid UTF-8 code unit sequence. In this case,
/// the number of bytes consumed is always between 0 and 3, inclusive, where
/// 0 is only returned when `slice` is empty.
///
/// # Examples
///
/// Basic usage:
///
/// ```
/// use bstr::decode_last_utf8;
///
/// // Decoding a valid codepoint.
/// let (ch, size) = decode_last_utf8(b"\xE2\x98\x83");
/// assert_eq!(Some('☃'), ch);
/// assert_eq!(3, size);
///
/// // Decoding an incomplete codepoint.
/// let (ch, size) = decode_last_utf8(b"\xE2\x98");
/// assert_eq!(None, ch);
/// assert_eq!(2, size);
/// ```
///
/// This example shows how to iterate over all codepoints in UTF-8 encoded
/// bytes in reverse, while replacing invalid UTF-8 sequences with the
/// replacement codepoint:
///
/// ```
/// use bstr::{B, decode_last_utf8};
///
/// let mut bytes = B(b"\xE2\x98\x83\xFF\xF0\x9D\x9E\x83\xE2\x98\x61");
/// let mut chars = vec![];
/// while !bytes.is_empty() {
///     let (ch, size) = decode_last_utf8(bytes);
///     bytes = &bytes[..bytes.len()-size];
///     chars.push(ch.unwrap_or('\u{FFFD}'));
/// }
/// assert_eq!(vec!['a', '\u{FFFD}', '��', '\u{FFFD}', '☃'], chars);
/// ```
#[inline]
pub fn decode_last<B: AsRef<[u8]>>(slice: B) -> (Option<char>, usize) {
    // TODO: We could implement this by reversing the UTF-8 automaton, but for
    // now, we do it the slow way by using the forward automaton.

    let slice = slice.as_ref();
    if slice.is_empty() {
        return (None, 0);
    }
    let mut start = slice.len() - 1;
    let limit = slice.len().saturating_sub(4);
    while start > limit && !is_leading_or_invalid_utf8_byte(slice[start]) {
        start -= 1;
    }
    let (ch, size) = decode(&slice[start..]);
    // If we didn't consume all of the bytes, then that means there's at least
    // one stray byte that never occurs in a valid code unit prefix, so we can
    // advance by one byte.
    if start + size != slice.len() {
        (None, 1)
    } else {
        (ch, size)
    }
}

/// Lossily UTF-8 decode a single Unicode scalar value from the end of a slice.
///
/// When successful, the corresponding Unicode scalar value is returned along
/// with the number of bytes it was encoded with. The number of bytes consumed
/// for a successful decode is always between 1 and 4, inclusive.
///
/// When unsuccessful, the Unicode replacement codepoint (`U+FFFD`) is returned
/// along with the number of bytes that make up a maximal prefix of a valid
/// UTF-8 code unit sequence. In this case, the number of bytes consumed is
/// always between 0 and 3, inclusive, where 0 is only returned when `slice` is
/// empty.
///
/// # Examples
///
/// Basic usage:
///
/// ```ignore
/// use bstr::decode_last_utf8_lossy;
///
/// // Decoding a valid codepoint.
/// let (ch, size) = decode_last_utf8_lossy(b"\xE2\x98\x83");
/// assert_eq!('☃', ch);
/// assert_eq!(3, size);
///
/// // Decoding an incomplete codepoint.
/// let (ch, size) = decode_last_utf8_lossy(b"\xE2\x98");
/// assert_eq!('\u{FFFD}', ch);
/// assert_eq!(2, size);
/// ```
///
/// This example shows how to iterate over all codepoints in UTF-8 encoded
/// bytes in reverse, while replacing invalid UTF-8 sequences with the
/// replacement codepoint:
///
/// ```ignore
/// use bstr::decode_last_utf8_lossy;
///
/// let mut bytes = B(b"\xE2\x98\x83\xFF\xF0\x9D\x9E\x83\xE2\x98\x61");
/// let mut chars = vec![];
/// while !bytes.is_empty() {
///     let (ch, size) = decode_last_utf8_lossy(bytes);
///     bytes = &bytes[..bytes.len()-size];
///     chars.push(ch);
/// }
/// assert_eq!(vec!['a', '\u{FFFD}', '��', '\u{FFFD}', '☃'], chars);
/// ```
#[inline]
pub fn decode_last_lossy<B: AsRef<[u8]>>(slice: B) -> (char, usize) {
    match decode_last(slice) {
        (Some(ch), size) => (ch, size),
        (None, size) => ('\u{FFFD}', size),
    }
}

/// SAFETY: The decode function relies on state being equal to ACCEPT only if
/// cp is a valid Unicode scalar value.
#[inline]
pub fn decode_step(state: &mut usize, cp: &mut u32, b: u8) {
    let class = CLASSES[b as usize];
    if *state == ACCEPT {
        *cp = (0xFF >> class) & (b as u32);
    } else {
        *cp = (b as u32 & 0b111111) | (*cp << 6);
    }
    *state = STATES_FORWARD[*state + class as usize] as usize;
}

/// Returns true if and only if the given byte is either a valid leading UTF-8
/// byte, or is otherwise an invalid byte that can never appear anywhere in a
/// valid UTF-8 sequence.
fn is_leading_or_invalid_utf8_byte(b: u8) -> bool {
    // In the ASCII case, the most significant bit is never set. The leading
    // byte of a 2/3/4-byte sequence always has the top two most significant
    // bits set. For bytes that can never appear anywhere in valid UTF-8, this
    // also returns true, since every such byte has its two most significant
    // bits set:
    //
    //     \xC0 :: 11000000
    //     \xC1 :: 11000001
    //     \xF5 :: 11110101
    //     \xF6 :: 11110110
    //     \xF7 :: 11110111
    //     \xF8 :: 11111000
    //     \xF9 :: 11111001
    //     \xFA :: 11111010
    //     \xFB :: 11111011
    //     \xFC :: 11111100
    //     \xFD :: 11111101
    //     \xFE :: 11111110
    //     \xFF :: 11111111
    (b & 0b1100_0000) != 0b1000_0000
}

#[cfg(test)]
mod tests {
    use std::char;

    use crate::ext_slice::{ByteSlice, B};
    use crate::tests::LOSSY_TESTS;
    use crate::utf8::{self, Utf8Error};

    fn utf8e(valid_up_to: usize) -> Utf8Error {
        Utf8Error { valid_up_to, error_len: None }
    }

    fn utf8e2(valid_up_to: usize, error_len: usize) -> Utf8Error {
        Utf8Error { valid_up_to, error_len: Some(error_len) }
    }

    #[test]
    fn validate_all_codepoints() {
        for i in 0..(0x10FFFF + 1) {
            let cp = match char::from_u32(i) {
                None => continue,
                Some(cp) => cp,
            };
            let mut buf = [0; 4];
            let s = cp.encode_utf8(&mut buf);
            assert_eq!(Ok(()), utf8::validate(s.as_bytes()));
        }
    }

    #[test]
    fn validate_multiple_codepoints() {
        assert_eq!(Ok(()), utf8::validate(b"abc"));
        assert_eq!(Ok(()), utf8::validate(b"a\xE2\x98\x83a"));
        assert_eq!(Ok(()), utf8::validate(b"a\xF0\x9D\x9C\xB7a"));
        assert_eq!(Ok(()), utf8::validate(b"\xE2\x98\x83\xF0\x9D\x9C\xB7",));
        assert_eq!(
            Ok(()),
            utf8::validate(b"a\xE2\x98\x83a\xF0\x9D\x9C\xB7a",)
        );
        assert_eq!(
            Ok(()),
            utf8::validate(b"\xEF\xBF\xBD\xE2\x98\x83\xEF\xBF\xBD",)
        );
    }

    #[test]
    fn validate_errors() {
        // single invalid byte
        assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xFF"));
        // single invalid byte after ASCII
        assert_eq!(Err(utf8e2(1, 1)), utf8::validate(b"a\xFF"));
        // single invalid byte after 2 byte sequence
        assert_eq!(Err(utf8e2(2, 1)), utf8::validate(b"\xCE\xB2\xFF"));
        // single invalid byte after 3 byte sequence
        assert_eq!(Err(utf8e2(3, 1)), utf8::validate(b"\xE2\x98\x83\xFF"));
        // single invalid byte after 4 byte sequence
        assert_eq!(Err(utf8e2(4, 1)), utf8::validate(b"\xF0\x9D\x9D\xB1\xFF"));

        // An invalid 2-byte sequence with a valid 1-byte prefix.
        assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xCE\xF0"));
        // An invalid 3-byte sequence with a valid 2-byte prefix.
        assert_eq!(Err(utf8e2(0, 2)), utf8::validate(b"\xE2\x98\xF0"));
        // An invalid 4-byte sequence with a valid 3-byte prefix.
        assert_eq!(Err(utf8e2(0, 3)), utf8::validate(b"\xF0\x9D\x9D\xF0"));

        // An overlong sequence. Should be \xE2\x82\xAC, but we encode the
        // same codepoint value in 4 bytes. This not only tests that we reject
        // overlong sequences, but that we get valid_up_to correct.
        assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xF0\x82\x82\xAC"));
        assert_eq!(Err(utf8e2(1, 1)), utf8::validate(b"a\xF0\x82\x82\xAC"));
        assert_eq!(
            Err(utf8e2(3, 1)),
            utf8::validate(b"\xE2\x98\x83\xF0\x82\x82\xAC",)
        );

        // Check that encoding a surrogate codepoint using the UTF-8 scheme
        // fails validation.
        assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xED\xA0\x80"));
        assert_eq!(Err(utf8e2(1, 1)), utf8::validate(b"a\xED\xA0\x80"));
        assert_eq!(
            Err(utf8e2(3, 1)),
            utf8::validate(b"\xE2\x98\x83\xED\xA0\x80",)
        );

        // Check that an incomplete 2-byte sequence fails.
        assert_eq!(Err(utf8e2(0, 1)), utf8::validate(b"\xCEa"));
        assert_eq!(Err(utf8e2(1, 1)), utf8::validate(b"a\xCEa"));
        assert_eq!(
            Err(utf8e2(3, 1)),
            utf8::validate(b"\xE2\x98\x83\xCE\xE2\x98\x83",)
        );
        // Check that an incomplete 3-byte sequence fails.
        assert_eq!(Err(utf8e2(0, 2)), utf8::validate(b"\xE2\x98a"));
        assert_eq!(Err(utf8e2(1, 2)), utf8::validate(b"a\xE2\x98a"));
        assert_eq!(
            Err(utf8e2(3, 2)),
            utf8::validate(b"\xE2\x98\x83\xE2\x98\xE2\x98\x83",)
        );
        // Check that an incomplete 4-byte sequence fails.
        assert_eq!(Err(utf8e2(0, 3)), utf8::validate(b"\xF0\x9D\x9Ca"));
        assert_eq!(Err(utf8e2(1, 3)), utf8::validate(b"a\xF0\x9D\x9Ca"));
        assert_eq!(
            Err(utf8e2(4, 3)),
            utf8::validate(b"\xF0\x9D\x9C\xB1\xF0\x9D\x9C\xE2\x98\x83",)
        );
        assert_eq!(
            Err(utf8e2(6, 3)),
            utf8::validate(b"foobar\xF1\x80\x80quux",)
        );

        // Check that an incomplete (EOF) 2-byte sequence fails.
        assert_eq!(Err(utf8e(0)), utf8::validate(b"\xCE"));
        assert_eq!(Err(utf8e(1)), utf8::validate(b"a\xCE"));
        assert_eq!(Err(utf8e(3)), utf8::validate(b"\xE2\x98\x83\xCE"));
        // Check that an incomplete (EOF) 3-byte sequence fails.
        assert_eq!(Err(utf8e(0)), utf8::validate(b"\xE2\x98"));
        assert_eq!(Err(utf8e(1)), utf8::validate(b"a\xE2\x98"));
        assert_eq!(Err(utf8e(3)), utf8::validate(b"\xE2\x98\x83\xE2\x98"));
        // Check that an incomplete (EOF) 4-byte sequence fails.
        assert_eq!(Err(utf8e(0)), utf8::validate(b"\xF0\x9D\x9C"));
        assert_eq!(Err(utf8e(1)), utf8::validate(b"a\xF0\x9D\x9C"));
        assert_eq!(
            Err(utf8e(4)),
            utf8::validate(b"\xF0\x9D\x9C\xB1\xF0\x9D\x9C",)
        );

        // Test that we errors correct even after long valid sequences. This
        // checks that our "backup" logic for detecting errors is correct.
        assert_eq!(
            Err(utf8e2(8, 1)),
            utf8::validate(b"\xe2\x98\x83\xce\xb2\xe3\x83\x84\xFF",)
        );
    }

    #[test]
    fn decode_valid() {
        fn d(mut s: &str) -> Vec<char> {
            let mut chars = vec![];
            while !s.is_empty() {
                let (ch, size) = utf8::decode(s.as_bytes());
                s = &s[size..];
                chars.push(ch.unwrap());
            }
            chars
        }

        assert_eq!(vec!['☃'], d("☃"));
        assert_eq!(vec!['☃', '☃'], d("☃☃"));
        assert_eq!(vec!['α', 'β', 'γ', 'δ', 'ε'], d("αβγδε"));
        assert_eq!(vec!['☃', '⛄', '⛇'], d("☃⛄⛇"));
        assert_eq!(vec!['��', '��', '��', '��', '��'], d("����������"));
    }

    #[test]
    fn decode_invalid() {
        let (ch, size) = utf8::decode(b"");
        assert_eq!(None, ch);
        assert_eq!(0, size);

        let (ch, size) = utf8::decode(b"\xFF");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode(b"\xCE\xF0");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode(b"\xE2\x98\xF0");
        assert_eq!(None, ch);
        assert_eq!(2, size);

        let (ch, size) = utf8::decode(b"\xF0\x9D\x9D");
        assert_eq!(None, ch);
        assert_eq!(3, size);

        let (ch, size) = utf8::decode(b"\xF0\x9D\x9D\xF0");
        assert_eq!(None, ch);
        assert_eq!(3, size);

        let (ch, size) = utf8::decode(b"\xF0\x82\x82\xAC");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode(b"\xED\xA0\x80");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode(b"\xCEa");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode(b"\xE2\x98a");
        assert_eq!(None, ch);
        assert_eq!(2, size);

        let (ch, size) = utf8::decode(b"\xF0\x9D\x9Ca");
        assert_eq!(None, ch);
        assert_eq!(3, size);
    }

    #[test]
    fn decode_lossy() {
        let (ch, size) = utf8::decode_lossy(b"");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(0, size);

        let (ch, size) = utf8::decode_lossy(b"\xFF");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_lossy(b"\xCE\xF0");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_lossy(b"\xE2\x98\xF0");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(2, size);

        let (ch, size) = utf8::decode_lossy(b"\xF0\x9D\x9D\xF0");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(3, size);

        let (ch, size) = utf8::decode_lossy(b"\xF0\x82\x82\xAC");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_lossy(b"\xED\xA0\x80");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_lossy(b"\xCEa");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_lossy(b"\xE2\x98a");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(2, size);

        let (ch, size) = utf8::decode_lossy(b"\xF0\x9D\x9Ca");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(3, size);
    }

    #[test]
    fn decode_last_valid() {
        fn d(mut s: &str) -> Vec<char> {
            let mut chars = vec![];
            while !s.is_empty() {
                let (ch, size) = utf8::decode_last(s.as_bytes());
                s = &s[..s.len() - size];
                chars.push(ch.unwrap());
            }
            chars
        }

        assert_eq!(vec!['☃'], d("☃"));
        assert_eq!(vec!['☃', '☃'], d("☃☃"));
        assert_eq!(vec!['ε', 'δ', 'γ', 'β', 'α'], d("αβγδε"));
        assert_eq!(vec!['⛇', '⛄', '☃'], d("☃⛄⛇"));
        assert_eq!(vec!['��', '��', '��', '��', '��'], d("����������"));
    }

    #[test]
    fn decode_last_invalid() {
        let (ch, size) = utf8::decode_last(b"");
        assert_eq!(None, ch);
        assert_eq!(0, size);

        let (ch, size) = utf8::decode_last(b"\xFF");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"\xCE\xF0");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"\xCE");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"\xE2\x98\xF0");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"\xE2\x98");
        assert_eq!(None, ch);
        assert_eq!(2, size);

        let (ch, size) = utf8::decode_last(b"\xF0\x9D\x9D\xF0");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"\xF0\x9D\x9D");
        assert_eq!(None, ch);
        assert_eq!(3, size);

        let (ch, size) = utf8::decode_last(b"\xF0\x82\x82\xAC");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"\xED\xA0\x80");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"\xED\xA0");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"\xED");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"a\xCE");
        assert_eq!(None, ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last(b"a\xE2\x98");
        assert_eq!(None, ch);
        assert_eq!(2, size);

        let (ch, size) = utf8::decode_last(b"a\xF0\x9D\x9C");
        assert_eq!(None, ch);
        assert_eq!(3, size);
    }

    #[test]
    fn decode_last_lossy() {
        let (ch, size) = utf8::decode_last_lossy(b"");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(0, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xFF");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xCE\xF0");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xCE");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xE2\x98\xF0");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xE2\x98");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(2, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xF0\x9D\x9D\xF0");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xF0\x9D\x9D");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(3, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xF0\x82\x82\xAC");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xED\xA0\x80");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xED\xA0");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"\xED");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"a\xCE");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(1, size);

        let (ch, size) = utf8::decode_last_lossy(b"a\xE2\x98");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(2, size);

        let (ch, size) = utf8::decode_last_lossy(b"a\xF0\x9D\x9C");
        assert_eq!('\u{FFFD}', ch);
        assert_eq!(3, size);
    }

    #[test]
    fn chars() {
        for (i, &(expected, input)) in LOSSY_TESTS.iter().enumerate() {
            let got: String = B(input).chars().collect();
            assert_eq!(
                expected, got,
                "chars(ith: {:?}, given: {:?})",
                i, input,
            );
            let got: String =
                B(input).char_indices().map(|(_, _, ch)| ch).collect();
            assert_eq!(
                expected, got,
                "char_indices(ith: {:?}, given: {:?})",
                i, input,
            );

            let expected: String = expected.chars().rev().collect();

            let got: String = B(input).chars().rev().collect();
            assert_eq!(
                expected, got,
                "chars.rev(ith: {:?}, given: {:?})",
                i, input,
            );
            let got: String =
                B(input).char_indices().rev().map(|(_, _, ch)| ch).collect();
            assert_eq!(
                expected, got,
                "char_indices.rev(ith: {:?}, given: {:?})",
                i, input,
            );
        }
    }

    #[test]
    fn utf8_chunks() {
        let mut c = utf8::Utf8Chunks { bytes: b"123\xC0" };
        assert_eq!(
            (c.next(), c.next()),
            (
                Some(utf8::Utf8Chunk {
                    valid: "123",
                    invalid: b"\xC0".as_bstr(),
                    incomplete: false,
                }),
                None,
            )
        );

        let mut c = utf8::Utf8Chunks { bytes: b"123\xFF\xFF" };
        assert_eq!(
            (c.next(), c.next(), c.next()),
            (
                Some(utf8::Utf8Chunk {
                    valid: "123",
                    invalid: b"\xFF".as_bstr(),
                    incomplete: false,
                }),
                Some(utf8::Utf8Chunk {
                    valid: "",
                    invalid: b"\xFF".as_bstr(),
                    incomplete: false,
                }),
                None,
            )
        );

        let mut c = utf8::Utf8Chunks { bytes: b"123\xD0" };
        assert_eq!(
            (c.next(), c.next()),
            (
                Some(utf8::Utf8Chunk {
                    valid: "123",
                    invalid: b"\xD0".as_bstr(),
                    incomplete: true,
                }),
                None,
            )
        );

        let mut c = utf8::Utf8Chunks { bytes: b"123\xD0456" };
        assert_eq!(
            (c.next(), c.next(), c.next()),
            (
                Some(utf8::Utf8Chunk {
                    valid: "123",
                    invalid: b"\xD0".as_bstr(),
                    incomplete: false,
                }),
                Some(utf8::Utf8Chunk {
                    valid: "456",
                    invalid: b"".as_bstr(),
                    incomplete: false,
                }),
                None,
            )
        );

        let mut c = utf8::Utf8Chunks { bytes: b"123\xE2\x98" };
        assert_eq!(
            (c.next(), c.next()),
            (
                Some(utf8::Utf8Chunk {
                    valid: "123",
                    invalid: b"\xE2\x98".as_bstr(),
                    incomplete: true,
                }),
                None,
            )
        );

        let mut c = utf8::Utf8Chunks { bytes: b"123\xF4\x8F\xBF" };
        assert_eq!(
            (c.next(), c.next()),
            (
                Some(utf8::Utf8Chunk {
                    valid: "123",
                    invalid: b"\xF4\x8F\xBF".as_bstr(),
                    incomplete: true,
                }),
                None,
            )
        );
    }
}