/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//! `aconfig_storage_file` is a crate that defines aconfig storage file format, it
//! also includes apis to read flags from storage files. It provides three apis to
//! interface with storage files:
//!
//! 1, function to get package flag value start offset
//! pub fn get_package_offset(container: &str, package: &str) -> `Result>>`
//!
//! 2, function to get flag offset within a specific package
//! pub fn get_flag_offset(container: &str, package_id: u32, flag: &str) -> `Result >>`
//!
//! 3, function to get the actual flag value given the global offset (combined package and
//! flag offset).
//! pub fn get_boolean_flag_value(container: &str, offset: u32) -> `Result`
//!
//! Note these are low level apis that are expected to be only used in auto generated flag
//! apis. DO NOT DIRECTLY USE THESE APIS IN YOUR SOURCE CODE. For auto generated flag apis
//! please refer to the g3doc go/android-flags
pub mod flag_info;
pub mod flag_table;
pub mod flag_value;
pub mod package_table;
pub mod protos;
pub mod sip_hasher13;
pub mod test_utils;
use anyhow::anyhow;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::fs::File;
use std::hash::Hasher;
use std::io::Read;
pub use crate::flag_info::{FlagInfoBit, FlagInfoHeader, FlagInfoList, FlagInfoNode};
pub use crate::flag_table::{FlagTable, FlagTableHeader, FlagTableNode};
pub use crate::flag_value::{FlagValueHeader, FlagValueList};
pub use crate::package_table::{PackageTable, PackageTableHeader, PackageTableNode};
pub use crate::sip_hasher13::SipHasher13;
use crate::AconfigStorageError::{
BytesParseFail, HashTableSizeLimit, InvalidFlagValueType, InvalidStoredFlagType,
};
/// The max storage file version from which we can safely read/write. May be
/// experimental.
pub const MAX_SUPPORTED_FILE_VERSION: u32 = 2;
/// The newest fully-released version. Unless otherwise specified, this is the
/// version we will write.
pub const DEFAULT_FILE_VERSION: u32 = 1;
/// Good hash table prime number
pub(crate) const HASH_PRIMES: [u32; 29] = [
7, 17, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, 196613, 393241,
786433, 1572869, 3145739, 6291469, 12582917, 25165843, 50331653, 100663319, 201326611,
402653189, 805306457, 1610612741,
];
/// Storage file type enum
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum StorageFileType {
PackageMap = 0,
FlagMap = 1,
FlagVal = 2,
FlagInfo = 3,
}
impl TryFrom<&str> for StorageFileType {
type Error = anyhow::Error;
fn try_from(value: &str) -> std::result::Result {
match value {
"package_map" => Ok(Self::PackageMap),
"flag_map" => Ok(Self::FlagMap),
"flag_val" => Ok(Self::FlagVal),
"flag_info" => Ok(Self::FlagInfo),
_ => Err(anyhow!(
"Invalid storage file type, valid types are package_map|flag_map|flag_val|flag_info"
)),
}
}
}
impl TryFrom for StorageFileType {
type Error = anyhow::Error;
fn try_from(value: u8) -> Result {
match value {
x if x == Self::PackageMap as u8 => Ok(Self::PackageMap),
x if x == Self::FlagMap as u8 => Ok(Self::FlagMap),
x if x == Self::FlagVal as u8 => Ok(Self::FlagVal),
x if x == Self::FlagInfo as u8 => Ok(Self::FlagInfo),
_ => Err(anyhow!("Invalid storage file type")),
}
}
}
/// Flag type enum as stored by storage file
/// ONLY APPEND, NEVER REMOVE FOR BACKWARD COMPATIBILITY. THE MAX IS U16.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum StoredFlagType {
ReadWriteBoolean = 0,
ReadOnlyBoolean = 1,
FixedReadOnlyBoolean = 2,
}
impl TryFrom for StoredFlagType {
type Error = AconfigStorageError;
fn try_from(value: u16) -> Result {
match value {
x if x == Self::ReadWriteBoolean as u16 => Ok(Self::ReadWriteBoolean),
x if x == Self::ReadOnlyBoolean as u16 => Ok(Self::ReadOnlyBoolean),
x if x == Self::FixedReadOnlyBoolean as u16 => Ok(Self::FixedReadOnlyBoolean),
_ => Err(InvalidStoredFlagType(anyhow!("Invalid stored flag type"))),
}
}
}
/// Flag value type enum, one FlagValueType maps to many StoredFlagType
/// ONLY APPEND, NEVER REMOVE FOR BACKWARD COMPATIBILITY. THE MAX IS U16
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FlagValueType {
Boolean = 0,
}
impl TryFrom for FlagValueType {
type Error = AconfigStorageError;
fn try_from(value: StoredFlagType) -> Result {
match value {
StoredFlagType::ReadWriteBoolean => Ok(Self::Boolean),
StoredFlagType::ReadOnlyBoolean => Ok(Self::Boolean),
StoredFlagType::FixedReadOnlyBoolean => Ok(Self::Boolean),
}
}
}
impl TryFrom for FlagValueType {
type Error = AconfigStorageError;
fn try_from(value: u16) -> Result {
match value {
x if x == Self::Boolean as u16 => Ok(Self::Boolean),
_ => Err(InvalidFlagValueType(anyhow!("Invalid flag value type"))),
}
}
}
/// Storage query api error
#[non_exhaustive]
#[derive(thiserror::Error, Debug)]
pub enum AconfigStorageError {
#[error("failed to read the file")]
FileReadFail(#[source] anyhow::Error),
#[error("fail to parse protobuf")]
ProtobufParseFail(#[source] anyhow::Error),
#[error("storage files not found for this container")]
StorageFileNotFound(#[source] anyhow::Error),
#[error("fail to map storage file")]
MapFileFail(#[source] anyhow::Error),
#[error("fail to get mapped file")]
ObtainMappedFileFail(#[source] anyhow::Error),
#[error("fail to flush mapped storage file")]
MapFlushFail(#[source] anyhow::Error),
#[error("number of items in hash table exceed limit")]
HashTableSizeLimit(#[source] anyhow::Error),
#[error("failed to parse bytes into data")]
BytesParseFail(#[source] anyhow::Error),
#[error("cannot parse storage files with a higher version")]
HigherStorageFileVersion(#[source] anyhow::Error),
#[error("invalid storage file byte offset")]
InvalidStorageFileOffset(#[source] anyhow::Error),
#[error("failed to create file")]
FileCreationFail(#[source] anyhow::Error),
#[error("invalid stored flag type")]
InvalidStoredFlagType(#[source] anyhow::Error),
#[error("invalid flag value type")]
InvalidFlagValueType(#[source] anyhow::Error),
}
/// Get the right hash table size given number of entries in the table. Use a
/// load factor of 0.5 for performance.
pub fn get_table_size(entries: u32) -> Result {
HASH_PRIMES
.iter()
.find(|&&num| num >= 2 * entries)
.copied()
.ok_or(HashTableSizeLimit(anyhow!("Number of items in a hash table exceeds limit")))
}
/// Get the corresponding bucket index given the key and number of buckets
pub(crate) fn get_bucket_index(val: &[u8], num_buckets: u32) -> u32 {
let mut s = SipHasher13::new();
s.write(val);
s.write_u8(0xff);
let ret = (s.finish() % num_buckets as u64) as u32;
ret
}
/// Read and parse bytes as u8
pub fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result {
let val =
u8::from_le_bytes(buf[*head..*head + 1].try_into().map_err(|errmsg| {
BytesParseFail(anyhow!("fail to parse u8 from bytes: {}", errmsg))
})?);
*head += 1;
Ok(val)
}
/// Read and parse bytes as u16
pub(crate) fn read_u16_from_bytes(
buf: &[u8],
head: &mut usize,
) -> Result {
let val =
u16::from_le_bytes(buf[*head..*head + 2].try_into().map_err(|errmsg| {
BytesParseFail(anyhow!("fail to parse u16 from bytes: {}", errmsg))
})?);
*head += 2;
Ok(val)
}
/// Read and parse the first 4 bytes of buf as u32.
pub fn read_u32_from_start_of_bytes(buf: &[u8]) -> Result {
read_u32_from_bytes(buf, &mut 0)
}
/// Read and parse bytes as u32
pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result {
let val =
u32::from_le_bytes(buf[*head..*head + 4].try_into().map_err(|errmsg| {
BytesParseFail(anyhow!("fail to parse u32 from bytes: {}", errmsg))
})?);
*head += 4;
Ok(val)
}
// Read and parse bytes as u64
pub fn read_u64_from_bytes(buf: &[u8], head: &mut usize) -> Result {
let val =
u64::from_le_bytes(buf[*head..*head + 8].try_into().map_err(|errmsg| {
BytesParseFail(anyhow!("fail to parse u64 from bytes: {}", errmsg))
})?);
*head += 8;
Ok(val)
}
/// Read and parse bytes as string
pub(crate) fn read_str_from_bytes(
buf: &[u8],
head: &mut usize,
) -> Result {
let num_bytes = read_u32_from_bytes(buf, head)? as usize;
let val = String::from_utf8(buf[*head..*head + num_bytes].to_vec())
.map_err(|errmsg| BytesParseFail(anyhow!("fail to parse string from bytes: {}", errmsg)))?;
*head += num_bytes;
Ok(val)
}
/// Read in storage file as bytes
pub fn read_file_to_bytes(file_path: &str) -> Result, AconfigStorageError> {
let mut file = File::open(file_path).map_err(|errmsg| {
AconfigStorageError::FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg))
})?;
let mut buffer = Vec::new();
file.read_to_end(&mut buffer).map_err(|errmsg| {
AconfigStorageError::FileReadFail(anyhow!(
"Failed to read bytes from file {}: {}",
file_path,
errmsg
))
})?;
Ok(buffer)
}
/// Flag value summary
#[derive(Debug, PartialEq)]
pub struct FlagValueSummary {
pub package_name: String,
pub flag_name: String,
pub flag_value: String,
pub value_type: StoredFlagType,
}
/// List flag values from storage files
pub fn list_flags(
package_map: &str,
flag_map: &str,
flag_val: &str,
) -> Result, AconfigStorageError> {
let package_table = PackageTable::from_bytes(&read_file_to_bytes(package_map)?)?;
let flag_table = FlagTable::from_bytes(&read_file_to_bytes(flag_map)?)?;
let flag_value_list = FlagValueList::from_bytes(&read_file_to_bytes(flag_val)?)?;
let mut package_info = vec![("", 0); package_table.header.num_packages as usize];
for node in package_table.nodes.iter() {
package_info[node.package_id as usize] = (&node.package_name, node.boolean_start_index);
}
let mut flags = Vec::new();
for node in flag_table.nodes.iter() {
let (package_name, boolean_start_index) = package_info[node.package_id as usize];
let flag_index = boolean_start_index + node.flag_index as u32;
let flag_value = flag_value_list.booleans[flag_index as usize];
flags.push(FlagValueSummary {
package_name: String::from(package_name),
flag_name: node.flag_name.clone(),
flag_value: flag_value.to_string(),
value_type: node.flag_type,
});
}
flags.sort_by(|v1, v2| match v1.package_name.cmp(&v2.package_name) {
Ordering::Equal => v1.flag_name.cmp(&v2.flag_name),
other => other,
});
Ok(flags)
}
/// Flag value and info summary
#[derive(Debug, PartialEq)]
pub struct FlagValueAndInfoSummary {
pub package_name: String,
pub flag_name: String,
pub flag_value: String,
pub value_type: StoredFlagType,
pub is_readwrite: bool,
pub has_server_override: bool,
pub has_local_override: bool,
}
/// List flag values and info from storage files
pub fn list_flags_with_info(
package_map: &str,
flag_map: &str,
flag_val: &str,
flag_info: &str,
) -> Result, AconfigStorageError> {
let package_table = PackageTable::from_bytes(&read_file_to_bytes(package_map)?)?;
let flag_table = FlagTable::from_bytes(&read_file_to_bytes(flag_map)?)?;
let flag_value_list = FlagValueList::from_bytes(&read_file_to_bytes(flag_val)?)?;
let flag_info = FlagInfoList::from_bytes(&read_file_to_bytes(flag_info)?)?;
let mut package_info = vec![("", 0); package_table.header.num_packages as usize];
for node in package_table.nodes.iter() {
package_info[node.package_id as usize] = (&node.package_name, node.boolean_start_index);
}
let mut flags = Vec::new();
for node in flag_table.nodes.iter() {
let (package_name, boolean_start_index) = package_info[node.package_id as usize];
let flag_index = boolean_start_index + node.flag_index as u32;
let flag_value = flag_value_list.booleans[flag_index as usize];
let flag_attribute = flag_info.nodes[flag_index as usize].attributes;
flags.push(FlagValueAndInfoSummary {
package_name: String::from(package_name),
flag_name: node.flag_name.clone(),
flag_value: flag_value.to_string(),
value_type: node.flag_type,
is_readwrite: flag_attribute & (FlagInfoBit::IsReadWrite as u8) != 0,
has_server_override: flag_attribute & (FlagInfoBit::HasServerOverride as u8) != 0,
has_local_override: flag_attribute & (FlagInfoBit::HasLocalOverride as u8) != 0,
});
}
flags.sort_by(|v1, v2| match v1.package_name.cmp(&v2.package_name) {
Ordering::Equal => v1.flag_name.cmp(&v2.flag_name),
other => other,
});
Ok(flags)
}
// *************************************** //
// CC INTERLOP
// *************************************** //
// Exported rust data structure and methods, c++ code will be generated
#[cxx::bridge]
mod ffi {
/// flag value summary cxx return
pub struct FlagValueSummaryCXX {
pub package_name: String,
pub flag_name: String,
pub flag_value: String,
pub value_type: String,
}
/// flag value and info summary cxx return
pub struct FlagValueAndInfoSummaryCXX {
pub package_name: String,
pub flag_name: String,
pub flag_value: String,
pub value_type: String,
pub is_readwrite: bool,
pub has_server_override: bool,
pub has_local_override: bool,
}
/// list flag result cxx return
pub struct ListFlagValueResultCXX {
pub query_success: bool,
pub error_message: String,
pub flags: Vec,
}
/// list flag with info result cxx return
pub struct ListFlagValueAndInfoResultCXX {
pub query_success: bool,
pub error_message: String,
pub flags: Vec,
}
// Rust export to c++
extern "Rust" {
pub fn list_flags_cxx(
package_map: &str,
flag_map: &str,
flag_val: &str,
) -> ListFlagValueResultCXX;
pub fn list_flags_with_info_cxx(
package_map: &str,
flag_map: &str,
flag_val: &str,
flag_info: &str,
) -> ListFlagValueAndInfoResultCXX;
}
}
/// implement flag value summary cxx return type
impl ffi::FlagValueSummaryCXX {
pub(crate) fn new(summary: FlagValueSummary) -> Self {
Self {
package_name: summary.package_name,
flag_name: summary.flag_name,
flag_value: summary.flag_value,
value_type: format!("{:?}", summary.value_type),
}
}
}
/// implement flag value and info summary cxx return type
impl ffi::FlagValueAndInfoSummaryCXX {
pub(crate) fn new(summary: FlagValueAndInfoSummary) -> Self {
Self {
package_name: summary.package_name,
flag_name: summary.flag_name,
flag_value: summary.flag_value,
value_type: format!("{:?}", summary.value_type),
is_readwrite: summary.is_readwrite,
has_server_override: summary.has_server_override,
has_local_override: summary.has_local_override,
}
}
}
/// implement list flag cxx interlop
pub fn list_flags_cxx(
package_map: &str,
flag_map: &str,
flag_val: &str,
) -> ffi::ListFlagValueResultCXX {
match list_flags(package_map, flag_map, flag_val) {
Ok(summary) => ffi::ListFlagValueResultCXX {
query_success: true,
error_message: String::new(),
flags: summary.into_iter().map(ffi::FlagValueSummaryCXX::new).collect(),
},
Err(errmsg) => ffi::ListFlagValueResultCXX {
query_success: false,
error_message: format!("{:?}", errmsg),
flags: Vec::new(),
},
}
}
/// implement list flag with info cxx interlop
pub fn list_flags_with_info_cxx(
package_map: &str,
flag_map: &str,
flag_val: &str,
flag_info: &str,
) -> ffi::ListFlagValueAndInfoResultCXX {
match list_flags_with_info(package_map, flag_map, flag_val, flag_info) {
Ok(summary) => ffi::ListFlagValueAndInfoResultCXX {
query_success: true,
error_message: String::new(),
flags: summary.into_iter().map(ffi::FlagValueAndInfoSummaryCXX::new).collect(),
},
Err(errmsg) => ffi::ListFlagValueAndInfoResultCXX {
query_success: false,
error_message: format!("{:?}", errmsg),
flags: Vec::new(),
},
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_utils::{
create_test_flag_info_list, create_test_flag_table, create_test_flag_value_list,
create_test_package_table, write_bytes_to_temp_file,
};
#[test]
// this test point locks down the flag list api
fn test_list_flag() {
let package_table =
write_bytes_to_temp_file(&create_test_package_table(DEFAULT_FILE_VERSION).into_bytes())
.unwrap();
let flag_table =
write_bytes_to_temp_file(&create_test_flag_table(DEFAULT_FILE_VERSION).into_bytes())
.unwrap();
let flag_value_list = write_bytes_to_temp_file(
&create_test_flag_value_list(DEFAULT_FILE_VERSION).into_bytes(),
)
.unwrap();
let package_table_path = package_table.path().display().to_string();
let flag_table_path = flag_table.path().display().to_string();
let flag_value_list_path = flag_value_list.path().display().to_string();
let flags =
list_flags(&package_table_path, &flag_table_path, &flag_value_list_path).unwrap();
let expected = [
FlagValueSummary {
package_name: String::from("com.android.aconfig.storage.test_1"),
flag_name: String::from("disabled_rw"),
value_type: StoredFlagType::ReadWriteBoolean,
flag_value: String::from("false"),
},
FlagValueSummary {
package_name: String::from("com.android.aconfig.storage.test_1"),
flag_name: String::from("enabled_ro"),
value_type: StoredFlagType::ReadOnlyBoolean,
flag_value: String::from("true"),
},
FlagValueSummary {
package_name: String::from("com.android.aconfig.storage.test_1"),
flag_name: String::from("enabled_rw"),
value_type: StoredFlagType::ReadWriteBoolean,
flag_value: String::from("true"),
},
FlagValueSummary {
package_name: String::from("com.android.aconfig.storage.test_2"),
flag_name: String::from("disabled_rw"),
value_type: StoredFlagType::ReadWriteBoolean,
flag_value: String::from("false"),
},
FlagValueSummary {
package_name: String::from("com.android.aconfig.storage.test_2"),
flag_name: String::from("enabled_fixed_ro"),
value_type: StoredFlagType::FixedReadOnlyBoolean,
flag_value: String::from("true"),
},
FlagValueSummary {
package_name: String::from("com.android.aconfig.storage.test_2"),
flag_name: String::from("enabled_ro"),
value_type: StoredFlagType::ReadOnlyBoolean,
flag_value: String::from("true"),
},
FlagValueSummary {
package_name: String::from("com.android.aconfig.storage.test_4"),
flag_name: String::from("enabled_fixed_ro"),
value_type: StoredFlagType::FixedReadOnlyBoolean,
flag_value: String::from("true"),
},
FlagValueSummary {
package_name: String::from("com.android.aconfig.storage.test_4"),
flag_name: String::from("enabled_rw"),
value_type: StoredFlagType::ReadWriteBoolean,
flag_value: String::from("true"),
},
];
assert_eq!(flags, expected);
}
#[test]
// this test point locks down the flag list with info api
fn test_list_flag_with_info() {
let package_table =
write_bytes_to_temp_file(&create_test_package_table(DEFAULT_FILE_VERSION).into_bytes())
.unwrap();
let flag_table =
write_bytes_to_temp_file(&create_test_flag_table(DEFAULT_FILE_VERSION).into_bytes())
.unwrap();
let flag_value_list = write_bytes_to_temp_file(
&create_test_flag_value_list(DEFAULT_FILE_VERSION).into_bytes(),
)
.unwrap();
let flag_info_list = write_bytes_to_temp_file(
&create_test_flag_info_list(DEFAULT_FILE_VERSION).into_bytes(),
)
.unwrap();
let package_table_path = package_table.path().display().to_string();
let flag_table_path = flag_table.path().display().to_string();
let flag_value_list_path = flag_value_list.path().display().to_string();
let flag_info_list_path = flag_info_list.path().display().to_string();
let flags = list_flags_with_info(
&package_table_path,
&flag_table_path,
&flag_value_list_path,
&flag_info_list_path,
)
.unwrap();
let expected = [
FlagValueAndInfoSummary {
package_name: String::from("com.android.aconfig.storage.test_1"),
flag_name: String::from("disabled_rw"),
value_type: StoredFlagType::ReadWriteBoolean,
flag_value: String::from("false"),
is_readwrite: true,
has_server_override: false,
has_local_override: false,
},
FlagValueAndInfoSummary {
package_name: String::from("com.android.aconfig.storage.test_1"),
flag_name: String::from("enabled_ro"),
value_type: StoredFlagType::ReadOnlyBoolean,
flag_value: String::from("true"),
is_readwrite: false,
has_server_override: false,
has_local_override: false,
},
FlagValueAndInfoSummary {
package_name: String::from("com.android.aconfig.storage.test_1"),
flag_name: String::from("enabled_rw"),
value_type: StoredFlagType::ReadWriteBoolean,
flag_value: String::from("true"),
is_readwrite: true,
has_server_override: false,
has_local_override: false,
},
FlagValueAndInfoSummary {
package_name: String::from("com.android.aconfig.storage.test_2"),
flag_name: String::from("disabled_rw"),
value_type: StoredFlagType::ReadWriteBoolean,
flag_value: String::from("false"),
is_readwrite: true,
has_server_override: false,
has_local_override: false,
},
FlagValueAndInfoSummary {
package_name: String::from("com.android.aconfig.storage.test_2"),
flag_name: String::from("enabled_fixed_ro"),
value_type: StoredFlagType::FixedReadOnlyBoolean,
flag_value: String::from("true"),
is_readwrite: false,
has_server_override: false,
has_local_override: false,
},
FlagValueAndInfoSummary {
package_name: String::from("com.android.aconfig.storage.test_2"),
flag_name: String::from("enabled_ro"),
value_type: StoredFlagType::ReadOnlyBoolean,
flag_value: String::from("true"),
is_readwrite: false,
has_server_override: false,
has_local_override: false,
},
FlagValueAndInfoSummary {
package_name: String::from("com.android.aconfig.storage.test_4"),
flag_name: String::from("enabled_fixed_ro"),
value_type: StoredFlagType::FixedReadOnlyBoolean,
flag_value: String::from("true"),
is_readwrite: false,
has_server_override: false,
has_local_override: false,
},
FlagValueAndInfoSummary {
package_name: String::from("com.android.aconfig.storage.test_4"),
flag_name: String::from("enabled_rw"),
value_type: StoredFlagType::ReadWriteBoolean,
flag_value: String::from("true"),
is_readwrite: true,
has_server_override: false,
has_local_override: false,
},
];
assert_eq!(flags, expected);
}
}