1 // Copyright 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 use crate::bitwidth::{align, BitWidth};
16 mod value;
17 use crate::FlexBufferType;
18 use std::cmp::max;
19 use value::{find_vector_type, store_value, Value};
20 mod map;
21 mod push;
22 mod ser;
23 mod vector;
24 use map::sort_map_by_keys;
25 pub use map::MapBuilder;
26 pub use push::Pushable;
27 pub use ser::{Error, FlexbufferSerializer};
28 pub use vector::VectorBuilder;
29
30 macro_rules! push_slice {
31 ($push_name: ident, $scalar: ty, $Val: ident, $new_vec: ident) => {
32 fn $push_name<T, S>(&mut self, xs: S)
33 where
34 T: Into<$scalar> + Copy,
35 S: AsRef<[T]>
36 {
37 let mut value = Value::$new_vec(xs.as_ref().len());
38 let mut width = xs.as_ref()
39 .iter()
40 .map(|x| BitWidth::from((*x).into()))
41 .max()
42 .unwrap_or_default();
43 if !value.is_fixed_length_vector() {
44 let length = Value::UInt(xs.as_ref().len() as u64);
45 width = std::cmp::max(width, length.width_or_child_width());
46 align(&mut self.buffer, width);
47 store_value(&mut self.buffer, length, width);
48 } else {
49 align(&mut self.buffer, width);
50 }
51 let address = self.buffer.len();
52 for &x in xs.as_ref().iter() {
53 store_value(&mut self.buffer, Value::$Val(x.into()), width);
54 }
55 value.set_address_or_panic(address);
56 value.set_child_width_or_panic(width);
57 self.values.push(value);
58 }
59 }
60 }
61 macro_rules! push_indirect {
62 ($push_name: ident, $scalar: ty, $Direct: ident, $Indirect: ident) => {
63 fn $push_name<T: Into<$scalar>>(&mut self, x: T) {
64 let x = Value::$Direct(x.into());
65 let child_width = x.width_or_child_width();
66 let address = self.buffer.len();
67 store_value(&mut self.buffer, x, child_width);
68 self.values.push(
69 Value::Reference {
70 address,
71 child_width,
72 fxb_type: FlexBufferType::$Indirect,
73 }
74 );
75 }
76 }
77 }
78
79 bitflags! {
80 /// Options for sharing data within a flexbuffer.
81 ///
82 /// These increase serialization time but decrease the size of the resulting buffer. By
83 /// default, `SHARE_KEYS`. You may wish to turn on `SHARE_STRINGS` if you know your data has
84 /// many duplicate strings or `SHARE_KEY_VECTORS` if your data has many maps with identical
85 /// keys.
86 ///
87 /// ## Not Yet Implemented
88 /// - `SHARE_STRINGS`
89 /// - `SHARE_KEY_VECTORS`
90 pub struct BuilderOptions: u8 {
91 const SHARE_NONE = 0;
92 const SHARE_KEYS = 1;
93 const SHARE_STRINGS = 2;
94 const SHARE_KEYS_AND_STRINGS = 3;
95 const SHARE_KEY_VECTORS = 4;
96 const SHARE_ALL = 7;
97 }
98 }
99 impl Default for BuilderOptions {
default() -> Self100 fn default() -> Self {
101 Self::SHARE_KEYS
102 }
103 }
104
105 #[derive(Debug, Clone, Copy)]
106 // Address of a Key inside of the buffer.
107 struct CachedKey(usize);
108
109 /// **Use this struct to build a Flexbuffer.**
110 ///
111 /// Flexbuffers may only have a single root value, which may be constructed
112 /// with one of the following functions.
113 /// * `build_singleton` will push 1 value to the buffer and serialize it as the root.
114 /// * `start_vector` returns a `VectorBuilder`, into which many (potentially
115 /// heterogenous) values can be pushed. The vector itself is the root and is serialized
116 /// when the `VectorBuilder` is dropped (or `end` is called).
117 /// * `start_map` returns a `MapBuilder`, which is similar to a `VectorBuilder` except
118 /// every value must be pushed with an associated key. The map is serialized when the
119 /// `MapBuilder` is dropped (or `end` is called).
120 ///
121 /// These functions reset and overwrite the Builder which means, while there are no
122 /// active `MapBuilder` or `VectorBuilder`, the internal buffer is empty or contains a
123 /// finished Flexbuffer. The internal buffer is accessed with `view`.
124 #[derive(Debug, Clone)]
125 pub struct Builder {
126 buffer: Vec<u8>,
127 values: Vec<Value>,
128 key_pool: Option<Vec<CachedKey>>,
129 }
130 impl Default for Builder {
default() -> Self131 fn default() -> Self {
132 let opts = Default::default();
133 Builder::new(opts)
134 }
135 }
136
137 impl<'a> Builder {
new(opts: BuilderOptions) -> Self138 pub fn new(opts: BuilderOptions) -> Self {
139 let key_pool = if opts.contains(BuilderOptions::SHARE_KEYS) {
140 Some(vec![])
141 } else {
142 None
143 };
144 Builder {
145 key_pool,
146 values: Vec::new(),
147 buffer: Vec::new(),
148 }
149 }
150 /// Shows the internal flexbuffer. It will either be empty or populated with the most
151 /// recently built flexbuffer.
view(&self) -> &[u8]152 pub fn view(&self) -> &[u8] {
153 &self.buffer
154 }
155 /// Returns the internal buffer, replacing it with a new vector. The returned buffer will
156 /// either be empty or populated with the most recently built flexbuffer.
take_buffer(&mut self) -> Vec<u8>157 pub fn take_buffer(&mut self) -> Vec<u8> {
158 let mut b = Vec::new();
159 std::mem::swap(&mut self.buffer, &mut b);
160 b
161 }
162 /// Resets the internal state. Automatically called before building a new flexbuffer.
reset(&mut self)163 pub fn reset(&mut self) {
164 self.buffer.clear();
165 self.values.clear();
166 if let Some(pool) = self.key_pool.as_mut() {
167 pool.clear();
168 }
169 }
push_key(&mut self, key: &str)170 fn push_key(&mut self, key: &str) {
171 debug_assert!(
172 key.bytes().all(|b| b != b'\0'),
173 "Keys must not have internal nulls."
174 );
175 // Search key pool if there is one.
176 let found = self.key_pool.as_ref().map(|pool| {
177 pool.binary_search_by(|&CachedKey(addr)| {
178 let old_key = map::get_key(&self.buffer, addr);
179 old_key.cloned().cmp(key.bytes())
180 })
181 });
182 let address = if let Some(Ok(idx)) = found {
183 // Found key in key pool.
184 self.key_pool.as_ref().unwrap()[idx].0
185 } else {
186 // Key not in pool (or no pool).
187 let address = self.buffer.len();
188 self.buffer.extend_from_slice(key.as_bytes());
189 self.buffer.push(b'\0');
190 address
191 };
192 if let Some(Err(idx)) = found {
193 // Insert into key pool.
194 let pool = self.key_pool.as_mut().unwrap();
195 pool.insert(idx, CachedKey(address));
196 }
197 self.values.push(Value::Key(address));
198 }
push_uint<T: Into<u64>>(&mut self, x: T)199 fn push_uint<T: Into<u64>>(&mut self, x: T) {
200 self.values.push(Value::UInt(x.into()));
201 }
push_int<T: Into<i64>>(&mut self, x: T)202 fn push_int<T: Into<i64>>(&mut self, x: T) {
203 self.values.push(Value::Int(x.into()));
204 }
push_float<T: Into<f64>>(&mut self, x: T)205 fn push_float<T: Into<f64>>(&mut self, x: T) {
206 self.values.push(Value::Float(x.into()));
207 }
push_null(&mut self)208 fn push_null(&mut self) {
209 self.values.push(Value::Null);
210 }
push_bool(&mut self, x: bool)211 fn push_bool(&mut self, x: bool) {
212 self.values.push(Value::Bool(x));
213 }
store_blob(&mut self, xs: &[u8]) -> Value214 fn store_blob(&mut self, xs: &[u8]) -> Value {
215 let length = Value::UInt(xs.len() as u64);
216 let width = length.width_or_child_width();
217 align(&mut self.buffer, width);
218 store_value(&mut self.buffer, length, width);
219 let address = self.buffer.len();
220 self.buffer.extend_from_slice(xs);
221 Value::Reference {
222 fxb_type: FlexBufferType::Blob,
223 address,
224 child_width: width,
225 }
226 }
push_str(&mut self, x: &str)227 fn push_str(&mut self, x: &str) {
228 let mut string = self.store_blob(x.as_bytes());
229 self.buffer.push(b'\0');
230 string.set_fxb_type_or_panic(FlexBufferType::String);
231 self.values.push(string);
232 }
push_blob(&mut self, x: &[u8])233 fn push_blob(&mut self, x: &[u8]) {
234 let blob = self.store_blob(x);
235 self.values.push(blob);
236 }
push_bools(&mut self, xs: &[bool])237 fn push_bools(&mut self, xs: &[bool]) {
238 let length = Value::UInt(xs.len() as u64);
239 let width = length.width_or_child_width();
240 align(&mut self.buffer, width);
241 store_value(&mut self.buffer, length, width);
242 let address = self.buffer.len();
243 for &b in xs.iter() {
244 self.buffer.push(b as u8);
245 self.buffer.resize(self.buffer.len() + width as usize, 0);
246 }
247 self.values.push(Value::Reference {
248 fxb_type: FlexBufferType::VectorBool,
249 address,
250 child_width: width,
251 });
252 }
253
254 push_slice!(push_uints, u64, UInt, new_uint_vector);
255 push_slice!(push_ints, i64, Int, new_int_vector);
256 push_slice!(push_floats, f64, Float, new_float_vector);
257 push_indirect!(push_indirect_int, i64, Int, IndirectInt);
258 push_indirect!(push_indirect_uint, u64, UInt, IndirectUInt);
259 push_indirect!(push_indirect_float, f64, Float, IndirectFloat);
260
261 /// Resets the builder and starts a new flexbuffer with a vector at the root.
262 /// The exact Flexbuffer vector type is dynamically inferred.
start_vector(&'a mut self) -> VectorBuilder<'a>263 pub fn start_vector(&'a mut self) -> VectorBuilder<'a> {
264 self.reset();
265 VectorBuilder {
266 builder: self,
267 start: None,
268 }
269 }
270 /// Resets the builder and builds a new flexbuffer with a map at the root.
start_map(&'a mut self) -> MapBuilder<'a>271 pub fn start_map(&'a mut self) -> MapBuilder<'a> {
272 self.reset();
273 MapBuilder {
274 builder: self,
275 start: None,
276 }
277 }
278 /// Resets the builder and builds a new flexbuffer with the pushed value at the root.
build_singleton<P: Pushable>(&mut self, p: P)279 pub fn build_singleton<P: Pushable>(&mut self, p: P) {
280 self.reset();
281 p.push_to_builder(self);
282 let root = self.values.pop().unwrap();
283 store_root(&mut self.buffer, root);
284 }
push<P: Pushable>(&mut self, p: P)285 fn push<P: Pushable>(&mut self, p: P) {
286 p.push_to_builder(self);
287 }
288 /// Stores the values past `previous_end` as a map or vector depending on `is_map`.
289 /// If `previous_end` is None then this was a root map / vector and the last value
290 /// is stored as the root.
end_map_or_vector(&mut self, is_map: bool, previous_end: Option<usize>)291 fn end_map_or_vector(&mut self, is_map: bool, previous_end: Option<usize>) {
292 let split = previous_end.unwrap_or(0);
293 let value = if is_map {
294 let key_vals = &mut self.values[split..];
295 sort_map_by_keys(key_vals, &self.buffer);
296 let key_vector = store_vector(&mut self.buffer, key_vals, StoreOption::MapKeys);
297 store_vector(&mut self.buffer, key_vals, StoreOption::Map(key_vector))
298 } else {
299 store_vector(&mut self.buffer, &self.values[split..], StoreOption::Vector)
300 };
301 self.values.truncate(split);
302 if previous_end.is_some() {
303 self.values.push(value);
304 } else {
305 store_root(&mut self.buffer, value);
306 }
307 }
308 }
309
310 /// Builds a Flexbuffer with the single pushed value as the root.
singleton<P: Pushable>(p: P) -> Vec<u8>311 pub fn singleton<P: Pushable>(p: P) -> Vec<u8> {
312 let mut b = Builder::default();
313 b.build_singleton(p);
314 let Builder { buffer, .. } = b;
315 buffer
316 }
317
318 /// Stores the root value, root type and root width.
319 /// This should be called to finish the Flexbuffer.
store_root(buffer: &mut Vec<u8>, root: Value)320 fn store_root(buffer: &mut Vec<u8>, root: Value) {
321 let root_width = root.width_in_vector(buffer.len(), 0);
322 align(buffer, root_width);
323 store_value(buffer, root, root_width);
324 buffer.push(root.packed_type(root_width));
325 buffer.push(root_width.n_bytes() as u8);
326 }
327
328 pub enum StoreOption {
329 Vector,
330 Map(Value),
331 MapKeys,
332 }
333 /// Writes a Flexbuffer Vector or Map.
334 /// StoreOption::Map(Keys) must be a Value::Key or this will panic.
335 // #[inline(always)]
store_vector(buffer: &mut Vec<u8>, values: &[Value], opt: StoreOption) -> Value336 pub fn store_vector(buffer: &mut Vec<u8>, values: &[Value], opt: StoreOption) -> Value {
337 let (skip, stride) = match opt {
338 StoreOption::Vector => (0, 1),
339 StoreOption::MapKeys => (0, 2),
340 StoreOption::Map(_) => (1, 2),
341 };
342 let iter_values = || values.iter().skip(skip).step_by(stride);
343
344 // Figure out vector type and how long is the prefix.
345 let mut result = if let StoreOption::Map(_) = opt {
346 Value::new_map()
347 } else {
348 find_vector_type(iter_values())
349 };
350 let length_slot = if !result.is_fixed_length_vector() {
351 let length = iter_values().count();
352 Some(Value::UInt(length as u64))
353 } else {
354 None
355 };
356 // Measure required width and align to it.
357 let mut width = BitWidth::W8;
358 if let StoreOption::Map(keys) = opt {
359 width = max(width, keys.width_in_vector(buffer.len(), 0))
360 }
361 if let Some(l) = length_slot {
362 width = max(width, l.width_or_child_width());
363 }
364 let prefix_length = result.prefix_length();
365 for (i, &val) in iter_values().enumerate() {
366 width = max(width, val.width_in_vector(buffer.len(), i + prefix_length));
367 }
368 align(buffer, width);
369 #[allow(deprecated)]
370 {
371 debug_assert_ne!(
372 result.fxb_type(),
373 FlexBufferType::VectorString,
374 "VectorString is deprecated and cannot be written.\
375 (https://github.com/google/flatbuffers/issues/5627)"
376 );
377 }
378 // Write Prefix.
379 if let StoreOption::Map(keys) = opt {
380 let key_width = Value::UInt(keys.width_or_child_width().n_bytes() as u64);
381 store_value(buffer, keys, width);
382 store_value(buffer, key_width, width);
383 }
384 if let Some(len) = length_slot {
385 store_value(buffer, len, width);
386 }
387 // Write data.
388 let address = buffer.len();
389 for &v in iter_values() {
390 store_value(buffer, v, width);
391 }
392 // Write types
393 if result.is_typed_vector_or_map() {
394 for v in iter_values() {
395 buffer.push(v.packed_type(width));
396 }
397 }
398 // Return Value representing this Vector.
399 result.set_address_or_panic(address);
400 result.set_child_width_or_panic(width);
401 result
402 }
403