• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //     https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 use crate::bitwidth::{align, BitWidth};
16 mod value;
17 use crate::FlexBufferType;
18 use std::cmp::max;
19 use value::{find_vector_type, store_value, Value};
20 mod map;
21 mod push;
22 mod ser;
23 mod vector;
24 use map::sort_map_by_keys;
25 pub use map::MapBuilder;
26 pub use push::Pushable;
27 pub use ser::{Error, FlexbufferSerializer};
28 pub use vector::VectorBuilder;
29 
30 macro_rules! push_slice {
31     ($push_name: ident, $scalar: ty, $Val: ident, $new_vec: ident) => {
32         fn $push_name<T, S>(&mut self, xs: S)
33         where
34             T: Into<$scalar> + Copy,
35             S: AsRef<[T]>
36         {
37             let mut value = Value::$new_vec(xs.as_ref().len());
38             let mut width = xs.as_ref()
39                 .iter()
40                 .map(|x| BitWidth::from((*x).into()))
41                 .max()
42                 .unwrap_or_default();
43             if !value.is_fixed_length_vector() {
44                 let length = Value::UInt(xs.as_ref().len() as u64);
45                 width = std::cmp::max(width, length.width_or_child_width());
46                 align(&mut self.buffer, width);
47                 store_value(&mut self.buffer, length, width);
48             } else {
49                 align(&mut self.buffer, width);
50             }
51             let address = self.buffer.len();
52             for &x in xs.as_ref().iter() {
53                 store_value(&mut self.buffer, Value::$Val(x.into()), width);
54             }
55             value.set_address_or_panic(address);
56             value.set_child_width_or_panic(width);
57             self.values.push(value);
58         }
59     }
60 }
61 macro_rules! push_indirect {
62     ($push_name: ident, $scalar: ty, $Direct: ident, $Indirect: ident) => {
63         fn $push_name<T: Into<$scalar>>(&mut self, x: T) {
64             let x = Value::$Direct(x.into());
65             let child_width = x.width_or_child_width();
66             let address = self.buffer.len();
67             store_value(&mut self.buffer, x, child_width);
68             self.values.push(
69                 Value::Reference {
70                     address,
71                     child_width,
72                     fxb_type: FlexBufferType::$Indirect,
73                 }
74             );
75         }
76     }
77 }
78 
79 bitflags! {
80     /// Options for sharing data within a flexbuffer.
81     ///
82     /// These increase serialization time but decrease the size of the resulting buffer. By
83     /// default, `SHARE_KEYS`. You may wish to turn on `SHARE_STRINGS` if you know your data has
84     /// many duplicate strings or `SHARE_KEY_VECTORS` if your data has many maps with identical
85     /// keys.
86     ///
87     /// ## Not Yet Implemented
88     /// - `SHARE_STRINGS`
89     /// - `SHARE_KEY_VECTORS`
90     pub struct BuilderOptions: u8 {
91         const SHARE_NONE = 0;
92         const SHARE_KEYS = 1;
93         const SHARE_STRINGS = 2;
94         const SHARE_KEYS_AND_STRINGS = 3;
95         const SHARE_KEY_VECTORS = 4;
96         const SHARE_ALL = 7;
97     }
98 }
99 impl Default for BuilderOptions {
default() -> Self100     fn default() -> Self {
101         Self::SHARE_KEYS
102     }
103 }
104 
105 #[derive(Debug, Clone, Copy)]
106 // Address of a Key inside of the buffer.
107 struct CachedKey(usize);
108 
109 /// **Use this struct to build a Flexbuffer.**
110 ///
111 /// Flexbuffers may only have a single root value, which may be constructed
112 /// with  one of the following functions.
113 /// * `build_singleton` will push 1 value to the buffer and serialize it as the root.
114 /// * `start_vector` returns a `VectorBuilder`, into which many (potentially
115 /// heterogenous) values can be pushed. The vector itself is the root and is serialized
116 /// when the `VectorBuilder` is dropped (or `end` is called).
117 /// * `start_map` returns a `MapBuilder`, which is similar to a `VectorBuilder` except
118 /// every value must be pushed with an associated key. The map is serialized when the
119 /// `MapBuilder` is dropped (or `end` is called).
120 ///
121 /// These functions reset and overwrite the Builder which means, while there are no
122 /// active `MapBuilder` or `VectorBuilder`, the internal buffer is empty or contains a
123 /// finished Flexbuffer. The internal buffer is accessed with `view`.
124 #[derive(Debug, Clone)]
125 pub struct Builder {
126     buffer: Vec<u8>,
127     values: Vec<Value>,
128     key_pool: Option<Vec<CachedKey>>,
129 }
130 impl Default for Builder {
default() -> Self131     fn default() -> Self {
132         let opts = Default::default();
133         Builder::new(opts)
134     }
135 }
136 
137 impl<'a> Builder {
new(opts: BuilderOptions) -> Self138     pub fn new(opts: BuilderOptions) -> Self {
139         let key_pool = if opts.contains(BuilderOptions::SHARE_KEYS) {
140             Some(vec![])
141         } else {
142             None
143         };
144         Builder {
145             key_pool,
146             values: Vec::new(),
147             buffer: Vec::new(),
148         }
149     }
150     /// Shows the internal flexbuffer. It will either be empty or populated with the most
151     /// recently built flexbuffer.
view(&self) -> &[u8]152     pub fn view(&self) -> &[u8] {
153         &self.buffer
154     }
155     /// Returns the internal buffer, replacing it with a new vector. The returned buffer will
156     /// either be empty or populated with the most recently built flexbuffer.
take_buffer(&mut self) -> Vec<u8>157     pub fn take_buffer(&mut self) -> Vec<u8> {
158         let mut b = Vec::new();
159         std::mem::swap(&mut self.buffer, &mut b);
160         b
161     }
162     /// Resets the internal state. Automatically called before building a new flexbuffer.
reset(&mut self)163     pub fn reset(&mut self) {
164         self.buffer.clear();
165         self.values.clear();
166         if let Some(pool) = self.key_pool.as_mut() {
167             pool.clear();
168         }
169     }
push_key(&mut self, key: &str)170     fn push_key(&mut self, key: &str) {
171         debug_assert!(
172             key.bytes().all(|b| b != b'\0'),
173             "Keys must not have internal nulls."
174         );
175         // Search key pool if there is one.
176         let found = self.key_pool.as_ref().map(|pool| {
177             pool.binary_search_by(|&CachedKey(addr)| {
178                 let old_key = map::get_key(&self.buffer, addr);
179                 old_key.cloned().cmp(key.bytes())
180             })
181         });
182         let address = if let Some(Ok(idx)) = found {
183             // Found key in key pool.
184             self.key_pool.as_ref().unwrap()[idx].0
185         } else {
186             // Key not in pool (or no pool).
187             let address = self.buffer.len();
188             self.buffer.extend_from_slice(key.as_bytes());
189             self.buffer.push(b'\0');
190             address
191         };
192         if let Some(Err(idx)) = found {
193             // Insert into key pool.
194             let pool = self.key_pool.as_mut().unwrap();
195             pool.insert(idx, CachedKey(address));
196         }
197         self.values.push(Value::Key(address));
198     }
push_uint<T: Into<u64>>(&mut self, x: T)199     fn push_uint<T: Into<u64>>(&mut self, x: T) {
200         self.values.push(Value::UInt(x.into()));
201     }
push_int<T: Into<i64>>(&mut self, x: T)202     fn push_int<T: Into<i64>>(&mut self, x: T) {
203         self.values.push(Value::Int(x.into()));
204     }
push_float<T: Into<f64>>(&mut self, x: T)205     fn push_float<T: Into<f64>>(&mut self, x: T) {
206         self.values.push(Value::Float(x.into()));
207     }
push_null(&mut self)208     fn push_null(&mut self) {
209         self.values.push(Value::Null);
210     }
push_bool(&mut self, x: bool)211     fn push_bool(&mut self, x: bool) {
212         self.values.push(Value::Bool(x));
213     }
store_blob(&mut self, xs: &[u8]) -> Value214     fn store_blob(&mut self, xs: &[u8]) -> Value {
215         let length = Value::UInt(xs.len() as u64);
216         let width = length.width_or_child_width();
217         align(&mut self.buffer, width);
218         store_value(&mut self.buffer, length, width);
219         let address = self.buffer.len();
220         self.buffer.extend_from_slice(xs);
221         Value::Reference {
222             fxb_type: FlexBufferType::Blob,
223             address,
224             child_width: width,
225         }
226     }
push_str(&mut self, x: &str)227     fn push_str(&mut self, x: &str) {
228         let mut string = self.store_blob(x.as_bytes());
229         self.buffer.push(b'\0');
230         string.set_fxb_type_or_panic(FlexBufferType::String);
231         self.values.push(string);
232     }
push_blob(&mut self, x: &[u8])233     fn push_blob(&mut self, x: &[u8]) {
234         let blob = self.store_blob(x);
235         self.values.push(blob);
236     }
push_bools(&mut self, xs: &[bool])237     fn push_bools(&mut self, xs: &[bool]) {
238         let length = Value::UInt(xs.len() as u64);
239         let width = length.width_or_child_width();
240         align(&mut self.buffer, width);
241         store_value(&mut self.buffer, length, width);
242         let address = self.buffer.len();
243         for &b in xs.iter() {
244             self.buffer.push(b as u8);
245             self.buffer.resize(self.buffer.len() + width as usize, 0);
246         }
247         self.values.push(Value::Reference {
248             fxb_type: FlexBufferType::VectorBool,
249             address,
250             child_width: width,
251         });
252     }
253 
254     push_slice!(push_uints, u64, UInt, new_uint_vector);
255     push_slice!(push_ints, i64, Int, new_int_vector);
256     push_slice!(push_floats, f64, Float, new_float_vector);
257     push_indirect!(push_indirect_int, i64, Int, IndirectInt);
258     push_indirect!(push_indirect_uint, u64, UInt, IndirectUInt);
259     push_indirect!(push_indirect_float, f64, Float, IndirectFloat);
260 
261     /// Resets the builder and starts a new flexbuffer with a vector at the root.
262     /// The exact Flexbuffer vector type is dynamically inferred.
start_vector(&'a mut self) -> VectorBuilder<'a>263     pub fn start_vector(&'a mut self) -> VectorBuilder<'a> {
264         self.reset();
265         VectorBuilder {
266             builder: self,
267             start: None,
268         }
269     }
270     /// Resets the builder and builds a new flexbuffer with a map at the root.
start_map(&'a mut self) -> MapBuilder<'a>271     pub fn start_map(&'a mut self) -> MapBuilder<'a> {
272         self.reset();
273         MapBuilder {
274             builder: self,
275             start: None,
276         }
277     }
278     /// Resets the builder and builds a new flexbuffer with the pushed value at the root.
build_singleton<P: Pushable>(&mut self, p: P)279     pub fn build_singleton<P: Pushable>(&mut self, p: P) {
280         self.reset();
281         p.push_to_builder(self);
282         let root = self.values.pop().unwrap();
283         store_root(&mut self.buffer, root);
284     }
push<P: Pushable>(&mut self, p: P)285     fn push<P: Pushable>(&mut self, p: P) {
286         p.push_to_builder(self);
287     }
288     /// Stores the values past `previous_end` as a map or vector depending on `is_map`.
289     /// If `previous_end` is None then this was a root map / vector and the last value
290     /// is stored as the root.
end_map_or_vector(&mut self, is_map: bool, previous_end: Option<usize>)291     fn end_map_or_vector(&mut self, is_map: bool, previous_end: Option<usize>) {
292         let split = previous_end.unwrap_or(0);
293         let value = if is_map {
294             let key_vals = &mut self.values[split..];
295             sort_map_by_keys(key_vals, &self.buffer);
296             let key_vector = store_vector(&mut self.buffer, key_vals, StoreOption::MapKeys);
297             store_vector(&mut self.buffer, key_vals, StoreOption::Map(key_vector))
298         } else {
299             store_vector(&mut self.buffer, &self.values[split..], StoreOption::Vector)
300         };
301         self.values.truncate(split);
302         if previous_end.is_some() {
303             self.values.push(value);
304         } else {
305             store_root(&mut self.buffer, value);
306         }
307     }
308 }
309 
310 /// Builds a Flexbuffer with the single pushed value as the root.
singleton<P: Pushable>(p: P) -> Vec<u8>311 pub fn singleton<P: Pushable>(p: P) -> Vec<u8> {
312     let mut b = Builder::default();
313     b.build_singleton(p);
314     let Builder { buffer, .. } = b;
315     buffer
316 }
317 
318 /// Stores the root value, root type and root width.
319 /// This should be called to finish the Flexbuffer.
store_root(buffer: &mut Vec<u8>, root: Value)320 fn store_root(buffer: &mut Vec<u8>, root: Value) {
321     let root_width = root.width_in_vector(buffer.len(), 0);
322     align(buffer, root_width);
323     store_value(buffer, root, root_width);
324     buffer.push(root.packed_type(root_width));
325     buffer.push(root_width.n_bytes() as u8);
326 }
327 
328 pub enum StoreOption {
329     Vector,
330     Map(Value),
331     MapKeys,
332 }
333 /// Writes a Flexbuffer Vector or Map.
334 /// StoreOption::Map(Keys) must be a Value::Key or this will panic.
335 // #[inline(always)]
store_vector(buffer: &mut Vec<u8>, values: &[Value], opt: StoreOption) -> Value336 pub fn store_vector(buffer: &mut Vec<u8>, values: &[Value], opt: StoreOption) -> Value {
337     let (skip, stride) = match opt {
338         StoreOption::Vector => (0, 1),
339         StoreOption::MapKeys => (0, 2),
340         StoreOption::Map(_) => (1, 2),
341     };
342     let iter_values = || values.iter().skip(skip).step_by(stride);
343 
344     // Figure out vector type and how long is the prefix.
345     let mut result = if let StoreOption::Map(_) = opt {
346         Value::new_map()
347     } else {
348         find_vector_type(iter_values())
349     };
350     let length_slot = if !result.is_fixed_length_vector() {
351         let length = iter_values().count();
352         Some(Value::UInt(length as u64))
353     } else {
354         None
355     };
356     // Measure required width and align to it.
357     let mut width = BitWidth::W8;
358     if let StoreOption::Map(keys) = opt {
359         width = max(width, keys.width_in_vector(buffer.len(), 0))
360     }
361     if let Some(l) = length_slot {
362         width = max(width, l.width_or_child_width());
363     }
364     let prefix_length = result.prefix_length();
365     for (i, &val) in iter_values().enumerate() {
366         width = max(width, val.width_in_vector(buffer.len(), i + prefix_length));
367     }
368     align(buffer, width);
369     #[allow(deprecated)]
370     {
371         debug_assert_ne!(
372             result.fxb_type(),
373             FlexBufferType::VectorString,
374             "VectorString is deprecated and cannot be written.\
375              (https://github.com/google/flatbuffers/issues/5627)"
376         );
377     }
378     // Write Prefix.
379     if let StoreOption::Map(keys) = opt {
380         let key_width = Value::UInt(keys.width_or_child_width().n_bytes() as u64);
381         store_value(buffer, keys, width);
382         store_value(buffer, key_width, width);
383     }
384     if let Some(len) = length_slot {
385         store_value(buffer, len, width);
386     }
387     // Write data.
388     let address = buffer.len();
389     for &v in iter_values() {
390         store_value(buffer, v, width);
391     }
392     // Write types
393     if result.is_typed_vector_or_map() {
394         for v in iter_values() {
395             buffer.push(v.packed_type(width));
396         }
397     }
398     // Return Value representing this Vector.
399     result.set_address_or_panic(address);
400     result.set_child_width_or_panic(width);
401     result
402 }
403