base/src/volatile_memory.rs

// Copyright 2017 The ChromiumOS Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

//! Types for volatile access to memory.
//!
//! Two of the core rules for safe rust is no data races and no aliased mutable references.
//! `VolatileSlice`, along with types that produce it which implement
//! `VolatileMemory`, allow us to sidestep that rule by wrapping pointers that absolutely have to be
//! accessed volatile. Some systems really do need to operate on shared memory and can't have the
//! compiler reordering or eliding access because it has no visibility into what other systems are
//! doing with that hunk of memory.
//!
//! For the purposes of maintaining safety, volatile memory has some rules of its own:
//! 1. No references or slices to volatile memory (`&` or `&mut`).
//! 2. Access should always been done with a volatile read or write.
//! The First rule is because having references of any kind to memory considered volatile would
//! violate pointer aliasing. The second is because unvolatile accesses are inherently undefined if
//! done concurrently without synchronization. With volatile access we know that the compiler has
//! not reordered or elided the access.

use std::cmp::min;
use std::mem::size_of;
use std::ptr::copy;
use std::ptr::read_volatile;
use std::ptr::write_bytes;
use std::ptr::write_volatile;
use std::result;
use std::slice;
use std::usize;

use remain::sorted;
use thiserror::Error;
use zerocopy::AsBytes;
use zerocopy::FromBytes;
use zerocopy::Ref;

use crate::IoBufMut;

#[sorted]
#[derive(Error, Eq, PartialEq, Debug)]
pub enum VolatileMemoryError {
    /// `addr` is out of bounds of the volatile memory slice.
    #[error("address 0x{addr:x} is out of bounds")]
    OutOfBounds { addr: usize },
    /// Taking a slice at `base` with `offset` would overflow `usize`.
    #[error("address 0x{base:x} offset by 0x{offset:x} would overflow")]
    Overflow { base: usize, offset: usize },
}

pub type VolatileMemoryResult<T> = result::Result<T, VolatileMemoryError>;

use crate::VolatileMemoryError as Error;
type Result<T> = VolatileMemoryResult<T>;

/// Trait for types that support raw volatile access to their data.
pub trait VolatileMemory {
    /// Gets a slice of memory at `offset` that is `count` bytes in length and supports volatile
    /// access.
    fn get_slice(&self, offset: usize, count: usize) -> Result<VolatileSlice>;
}

/// A slice of raw memory that supports volatile access. Like `std::io::IoSliceMut`, this type is
/// guaranteed to be ABI-compatible with `libc::iovec` but unlike `IoSliceMut`, it doesn't
/// automatically deref to `&mut [u8]`.
#[derive(Copy, Clone, Debug)]
#[repr(transparent)]
pub struct VolatileSlice<'a>(IoBufMut<'a>);

impl<'a> VolatileSlice<'a> {
    /// Creates a slice of raw memory that must support volatile access.
    pub fn new(buf: &mut [u8]) -> VolatileSlice {
        VolatileSlice(IoBufMut::new(buf))
    }

    /// Creates a `VolatileSlice` from a pointer and a length.
    ///
    /// # Safety
    ///
    /// In order to use this method safely, `addr` must be valid for reads and writes of `len` bytes
    /// and should live for the entire duration of lifetime `'a`.
    pub unsafe fn from_raw_parts(addr: *mut u8, len: usize) -> VolatileSlice<'a> {
        VolatileSlice(IoBufMut::from_raw_parts(addr, len))
    }

    /// Gets a const pointer to this slice's memory.
    pub fn as_ptr(&self) -> *const u8 {
        self.0.as_ptr()
    }

    /// Gets a mutable pointer to this slice's memory.
    pub fn as_mut_ptr(&self) -> *mut u8 {
        self.0.as_mut_ptr()
    }

    /// Gets the size of this slice.
    pub fn size(&self) -> usize {
        self.0.len()
    }

    /// Advance the starting position of this slice.
    ///
    /// Panics if `count > self.size()`.
    pub fn advance(&mut self, count: usize) {
        self.0.advance(count)
    }

    /// Shorten the length of the slice.
    ///
    /// Has no effect if `len > self.size()`.
    pub fn truncate(&mut self, len: usize) {
        self.0.truncate(len)
    }

    /// Returns this `VolatileSlice` as an `IoBufMut`.
    pub fn as_iobuf(&self) -> &IoBufMut {
        &self.0
    }

    /// Converts a slice of `VolatileSlice`s into a slice of `IoBufMut`s
    #[allow(clippy::wrong_self_convention)]
    pub fn as_iobufs<'mem, 'slice>(
        iovs: &'slice [VolatileSlice<'mem>],
    ) -> &'slice [IoBufMut<'mem>] {
        // SAFETY:
        // Safe because `VolatileSlice` is ABI-compatible with `IoBufMut`.
        unsafe { slice::from_raw_parts(iovs.as_ptr() as *const IoBufMut, iovs.len()) }
    }

    /// Converts a mutable slice of `VolatileSlice`s into a mutable slice of `IoBufMut`s
    #[inline]
    pub fn as_iobufs_mut<'mem, 'slice>(
        iovs: &'slice mut [VolatileSlice<'mem>],
    ) -> &'slice mut [IoBufMut<'mem>] {
        // SAFETY:
        // Safe because `VolatileSlice` is ABI-compatible with `IoBufMut`.
        unsafe { slice::from_raw_parts_mut(iovs.as_mut_ptr() as *mut IoBufMut, iovs.len()) }
    }

    /// Creates a copy of this slice with the address increased by `count` bytes, and the size
    /// reduced by `count` bytes.
    pub fn offset(self, count: usize) -> Result<VolatileSlice<'a>> {
        let new_addr = (self.as_mut_ptr() as usize).checked_add(count).ok_or(
            VolatileMemoryError::Overflow {
                base: self.as_mut_ptr() as usize,
                offset: count,
            },
        )?;
        let new_size = self
            .size()
            .checked_sub(count)
            .ok_or(VolatileMemoryError::OutOfBounds { addr: new_addr })?;

        // SAFETY:
        // Safe because the memory has the same lifetime and points to a subset of the memory of the
        // original slice.
        unsafe { Ok(VolatileSlice::from_raw_parts(new_addr as *mut u8, new_size)) }
    }

    /// Similar to `get_slice` but the returned slice outlives this slice.
    ///
    /// The returned slice's lifetime is still limited by the underlying data's lifetime.
    pub fn sub_slice(self, offset: usize, count: usize) -> Result<VolatileSlice<'a>> {
        let mem_end = offset
            .checked_add(count)
            .ok_or(VolatileMemoryError::Overflow {
                base: offset,
                offset: count,
            })?;
        if mem_end > self.size() {
            return Err(Error::OutOfBounds { addr: mem_end });
        }
        let new_addr = (self.as_mut_ptr() as usize).checked_add(offset).ok_or(
            VolatileMemoryError::Overflow {
                base: self.as_mut_ptr() as usize,
                offset,
            },
        )?;

        // SAFETY:
        // Safe because we have verified that the new memory is a subset of the original slice.
        Ok(unsafe { VolatileSlice::from_raw_parts(new_addr as *mut u8, count) })
    }

    /// Sets each byte of this slice with the given byte, similar to `memset`.
    ///
    /// The bytes of this slice are accessed in an arbitray order.
    ///
    /// # Examples
    ///
    /// ```
    /// # use base::VolatileSlice;
    /// # fn test_write_45() -> Result<(), ()> {
    /// let mut mem = [0u8; 32];
    /// let vslice = VolatileSlice::new(&mut mem[..]);
    /// vslice.write_bytes(45);
    /// for &v in &mem[..] {
    ///     assert_eq!(v, 45);
    /// }
    /// # Ok(())
    /// # }
    pub fn write_bytes(&self, value: u8) {
        // SAFETY:
        // Safe because the memory is valid and needs only byte alignment.
        unsafe {
            write_bytes(self.as_mut_ptr(), value, self.size());
        }
    }

    /// Copies `self.size()` or `buf.len()` times the size of `T` bytes, whichever is smaller, to
    /// `buf`.
    ///
    /// The copy happens from smallest to largest address in `T` sized chunks using volatile reads.
    ///
    /// # Examples
    ///
    /// ```
    /// # use std::fs::File;
    /// # use std::path::Path;
    /// # use base::VolatileSlice;
    /// # fn test_write_null() -> Result<(), ()> {
    /// let mut mem = [0u8; 32];
    /// let vslice = VolatileSlice::new(&mut mem[..]);
    /// let mut buf = [5u8; 16];
    /// vslice.copy_to(&mut buf[..]);
    /// for v in &buf[..] {
    ///     assert_eq!(buf[0], 0);
    /// }
    /// # Ok(())
    /// # }
    /// ```
    pub fn copy_to<T>(&self, buf: &mut [T])
    where
        T: FromBytes + AsBytes + Copy,
    {
        let mut addr = self.as_mut_ptr() as *const u8;
        for v in buf.iter_mut().take(self.size() / size_of::<T>()) {
            // SAFETY: Safe because buf is valid, aligned to type `T` and is initialized.
            unsafe {
                *v = read_volatile(addr as *const T);
                addr = addr.add(size_of::<T>());
            }
        }
    }

    /// Copies `self.size()` or `slice.size()` bytes, whichever is smaller, to `slice`.
    ///
    /// The copies happen in an undefined order.
    /// # Examples
    ///
    /// ```
    /// # use base::VolatileMemory;
    /// # use base::VolatileSlice;
    /// # fn test_write_null() -> Result<(), ()> {
    /// let mut mem = [0u8; 32];
    /// let vslice = VolatileSlice::new(&mut mem[..]);
    /// vslice.copy_to_volatile_slice(vslice.get_slice(16, 16).map_err(|_| ())?);
    /// # Ok(())
    /// # }
    /// ```
    pub fn copy_to_volatile_slice(&self, slice: VolatileSlice) {
        // SAFETY: Safe because slice is valid and is byte aligned.
        unsafe {
            copy(
                self.as_mut_ptr() as *const u8,
                slice.as_mut_ptr(),
                min(self.size(), slice.size()),
            );
        }
    }

    /// Copies `self.size()` or `buf.len()` times the size of `T` bytes, whichever is smaller, to
    /// this slice's memory.
    ///
    /// The copy happens from smallest to largest address in `T` sized chunks using volatile writes.
    ///
    /// # Examples
    ///
    /// ```
    /// # use std::fs::File;
    /// # use std::path::Path;
    /// # use base::VolatileMemory;
    /// # use base::VolatileSlice;
    /// # fn test_write_null() -> Result<(), ()> {
    /// let mut mem = [0u8; 32];
    /// let vslice = VolatileSlice::new(&mut mem[..]);
    /// let buf = [5u8; 64];
    /// vslice.copy_from(&buf[..]);
    /// let mut copy_buf = [0u32; 4];
    /// vslice.copy_to(&mut copy_buf);
    /// for i in 0..4 {
    ///     assert_eq!(copy_buf[i], 0x05050505);
    /// }
    /// # Ok(())
    /// # }
    /// ```
    pub fn copy_from<T>(&self, buf: &[T])
    where
        T: FromBytes + AsBytes,
    {
        let mut addr = self.as_mut_ptr();
        for v in buf.iter().take(self.size() / size_of::<T>()) {
            // SAFETY: Safe because buf is valid, aligned to type `T` and is mutable.
            unsafe {
                write_volatile(
                    addr as *mut T,
                    Ref::<_, T>::new(v.as_bytes()).unwrap().read(),
                );
                addr = addr.add(size_of::<T>());
            }
        }
    }

    /// Returns whether all bytes in this slice are zero or not.
    ///
    /// This is optimized for [VolatileSlice] aligned with 16 bytes.
    ///
    /// TODO(b/274840085): Use SIMD for better performance.
    pub fn is_all_zero(&self) -> bool {
        const MASK_4BIT: usize = 0x0f;
        let head_addr = self.as_ptr() as usize;
        // Round up by 16
        let aligned_head_addr = (head_addr + MASK_4BIT) & !MASK_4BIT;
        let tail_addr = head_addr + self.size();
        // Round down by 16
        let aligned_tail_addr = tail_addr & !MASK_4BIT;

        // Check 16 bytes at once. The addresses should be 16 bytes aligned for better performance.
        if (aligned_head_addr..aligned_tail_addr).step_by(16).any(
            |aligned_addr|
                // SAFETY: Each aligned_addr is within VolatileSlice
                unsafe { *(aligned_addr as *const u128) } != 0,
        ) {
            return false;
        }

        if head_addr == aligned_head_addr && tail_addr == aligned_tail_addr {
            // If head_addr and tail_addr are aligned, we can skip the unaligned part which contains
            // at least 2 conditional branches.
            true
        } else {
            // Check unaligned part.
            // SAFETY: The range [head_addr, aligned_head_addr) and [aligned_tail_addr, tail_addr)
            // are within VolatileSlice.
            unsafe {
                is_all_zero_naive(head_addr, aligned_head_addr)
                    && is_all_zero_naive(aligned_tail_addr, tail_addr)
            }
        }
    }
}

/// Check whether every byte is zero.
///
/// This checks byte by byte.
///
/// # Safety
///
/// * `head_addr` <= `tail_addr`
/// * Bytes between `head_addr` and `tail_addr` is valid to access.
unsafe fn is_all_zero_naive(head_addr: usize, tail_addr: usize) -> bool {
    (head_addr..tail_addr).all(|addr| *(addr as *const u8) == 0)
}

impl<'a> VolatileMemory for VolatileSlice<'a> {
    fn get_slice(&self, offset: usize, count: usize) -> Result<VolatileSlice> {
        self.sub_slice(offset, count)
    }
}

impl PartialEq<VolatileSlice<'_>> for VolatileSlice<'_> {
    fn eq(&self, other: &VolatileSlice) -> bool {
        let size = self.size();
        if size != other.size() {
            return false;
        }

        // SAFETY: We pass pointers into valid VolatileSlice regions, and size is checked above.
        let cmp = unsafe { libc::memcmp(self.as_ptr() as _, other.as_ptr() as _, size) };

        cmp == 0
    }
}

/// The `PartialEq` implementation for `VolatileSlice` is reflexive, symmetric, and transitive.
impl Eq for VolatileSlice<'_> {}

#[cfg(test)]
mod tests {
    use std::sync::Arc;
    use std::sync::Barrier;
    use std::thread::spawn;

    use super::*;

    #[derive(Clone)]
    struct VecMem {
        mem: Arc<Vec<u8>>,
    }

    impl VecMem {
        fn new(size: usize) -> VecMem {
            VecMem {
                mem: Arc::new(vec![0u8; size]),
            }
        }
    }

    impl VolatileMemory for VecMem {
        fn get_slice(&self, offset: usize, count: usize) -> Result<VolatileSlice> {
            let mem_end = offset
                .checked_add(count)
                .ok_or(VolatileMemoryError::Overflow {
                    base: offset,
                    offset: count,
                })?;
            if mem_end > self.mem.len() {
                return Err(Error::OutOfBounds { addr: mem_end });
            }

            let new_addr = (self.mem.as_ptr() as usize).checked_add(offset).ok_or(
                VolatileMemoryError::Overflow {
                    base: self.mem.as_ptr() as usize,
                    offset,
                },
            )?;

            Ok(
                // SAFETY: trivially safe
                unsafe { VolatileSlice::from_raw_parts(new_addr as *mut u8, count) },
            )
        }
    }

    #[test]
    fn observe_mutate() {
        let a = VecMem::new(1);
        let a_clone = a.clone();
        a.get_slice(0, 1).unwrap().write_bytes(99);

        let start_barrier = Arc::new(Barrier::new(2));
        let thread_start_barrier = start_barrier.clone();
        let end_barrier = Arc::new(Barrier::new(2));
        let thread_end_barrier = end_barrier.clone();
        spawn(move || {
            thread_start_barrier.wait();
            a_clone.get_slice(0, 1).unwrap().write_bytes(0);
            thread_end_barrier.wait();
        });

        let mut byte = [0u8; 1];
        a.get_slice(0, 1).unwrap().copy_to(&mut byte);
        assert_eq!(byte[0], 99);

        start_barrier.wait();
        end_barrier.wait();

        a.get_slice(0, 1).unwrap().copy_to(&mut byte);
        assert_eq!(byte[0], 0);
    }

    #[test]
    fn slice_size() {
        let a = VecMem::new(100);
        let s = a.get_slice(0, 27).unwrap();
        assert_eq!(s.size(), 27);

        let s = a.get_slice(34, 27).unwrap();
        assert_eq!(s.size(), 27);

        let s = s.get_slice(20, 5).unwrap();
        assert_eq!(s.size(), 5);
    }

    #[test]
    fn slice_overflow_error() {
        use std::usize::MAX;
        let a = VecMem::new(1);
        let res = a.get_slice(MAX, 1).unwrap_err();
        assert_eq!(
            res,
            Error::Overflow {
                base: MAX,
                offset: 1,
            }
        );
    }

    #[test]
    fn slice_oob_error() {
        let a = VecMem::new(100);
        a.get_slice(50, 50).unwrap();
        let res = a.get_slice(55, 50).unwrap_err();
        assert_eq!(res, Error::OutOfBounds { addr: 105 });
    }

    #[test]
    fn is_all_zero_16bytes_aligned() {
        let a = VecMem::new(1024);
        let slice = a.get_slice(0, 1024).unwrap();

        assert!(slice.is_all_zero());
        a.get_slice(129, 1).unwrap().write_bytes(1);
        assert!(!slice.is_all_zero());
    }

    #[test]
    fn is_all_zero_head_not_aligned() {
        let a = VecMem::new(1024);
        let slice = a.get_slice(1, 1023).unwrap();

        assert!(slice.is_all_zero());
        a.get_slice(0, 1).unwrap().write_bytes(1);
        assert!(slice.is_all_zero());
        a.get_slice(1, 1).unwrap().write_bytes(1);
        assert!(!slice.is_all_zero());
        a.get_slice(1, 1).unwrap().write_bytes(0);
        a.get_slice(129, 1).unwrap().write_bytes(1);
        assert!(!slice.is_all_zero());
    }

    #[test]
    fn is_all_zero_tail_not_aligned() {
        let a = VecMem::new(1024);
        let slice = a.get_slice(0, 1023).unwrap();

        assert!(slice.is_all_zero());
        a.get_slice(1023, 1).unwrap().write_bytes(1);
        assert!(slice.is_all_zero());
        a.get_slice(1022, 1).unwrap().write_bytes(1);
        assert!(!slice.is_all_zero());
        a.get_slice(1022, 1).unwrap().write_bytes(0);
        a.get_slice(0, 1).unwrap().write_bytes(1);
        assert!(!slice.is_all_zero());
    }

    #[test]
    fn is_all_zero_no_aligned_16bytes() {
        let a = VecMem::new(1024);
        let slice = a.get_slice(1, 16).unwrap();

        assert!(slice.is_all_zero());
        a.get_slice(0, 1).unwrap().write_bytes(1);
        assert!(slice.is_all_zero());
        for i in 1..17 {
            a.get_slice(i, 1).unwrap().write_bytes(1);
            assert!(!slice.is_all_zero());
            a.get_slice(i, 1).unwrap().write_bytes(0);
        }
        a.get_slice(17, 1).unwrap().write_bytes(1);
        assert!(slice.is_all_zero());
    }
}