1 // Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>
2
3 // Use, modification and distribution is subject to the Boost Software
4 // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
5 // http://www.boost.org/LICENSE_1_0.txt)
6
7 // Authors: Douglas Gregor
8
9 /** @file serialize.hpp
10 *
11 * This file provides Boost.Serialization support for Python objects
12 * within Boost.MPI. Python objects can be serialized in one of two
13 * ways. The default serialization method involves using the Python
14 * "pickle" module to pickle the Python objects, transmits the
15 * pickled representation, and unpickles the result when
16 * received. For C++ types that have been exposed to Python and
17 * registered with register_serialized(), objects are directly
18 * serialized for transmissing, skipping the pickling step.
19 */
20 #ifndef BOOST_MPI_PYTHON_SERIALIZE_HPP
21 #define BOOST_MPI_PYTHON_SERIALIZE_HPP
22
23 #include <boost/mpi/python/config.hpp>
24
25 #include <boost/python/object.hpp>
26 #include <boost/python/str.hpp>
27 #include <boost/python/extract.hpp>
28
29 #include <map>
30
31 #include <boost/function/function3.hpp>
32
33 #include <boost/mpl/bool.hpp>
34 #include <boost/mpl/if.hpp>
35
36 #include <boost/serialization/split_free.hpp>
37 #include <boost/serialization/array.hpp>
38 #include <boost/serialization/array_wrapper.hpp>
39 #include <boost/smart_ptr/scoped_array.hpp>
40
41 #include <boost/assert.hpp>
42
43 #include <boost/type_traits/is_fundamental.hpp>
44
45 #define BOOST_MPI_PYTHON_FORWARD_ONLY
46 #include <boost/mpi/python.hpp>
47
48 #include "bytesobject.h"
49
50 /************************************************************************
51 * Boost.Python Serialization Section *
52 ************************************************************************/
53 #if !defined(BOOST_NO_SFINAE) && !defined(BOOST_NO_IS_CONVERTIBLE)
54 /**
55 * @brief Declare IArchive and OArchive as a Boost.Serialization
56 * archives that can be used for Python objects.
57 *
58 * This macro can only be expanded from the global namespace. It only
59 * requires that Archiver be forward-declared. IArchiver and OArchiver
60 * will only support Serialization of Python objects by pickling
61 * them. If the Archiver type should also support "direct"
62 * serialization (for C++ types), use
63 * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE instead.
64 */
65 # define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \
66 namespace boost { namespace python { namespace api { \
67 template<typename R, typename T> \
68 struct enable_binary< IArchiver , R, T> {}; \
69 \
70 template<typename R, typename T> \
71 struct enable_binary< OArchiver , R, T> {}; \
72 } } }
73 # else
74 # define BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver)
75 #endif
76
77 /**
78 * @brief Declare IArchiver and OArchiver as a Boost.Serialization
79 * archives that can be used for Python objects and C++ objects
80 * wrapped in Python.
81 *
82 * This macro can only be expanded from the global namespace. It only
83 * requires that IArchiver and OArchiver be forward-declared. However,
84 * note that you will also need to write
85 * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver,
86 * OArchiver) in one of your translation units.
87
88 DPG PICK UP HERE
89 */
90 #define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \
91 BOOST_PYTHON_SERIALIZATION_ARCHIVE(IArchiver, OArchiver) \
92 namespace boost { namespace python { namespace detail { \
93 template<> \
94 BOOST_MPI_PYTHON_DECL direct_serialization_table< IArchiver , OArchiver >& \
95 get_direct_serialization_table< IArchiver , OArchiver >(); \
96 } \
97 \
98 template<> \
99 struct has_direct_serialization< IArchiver , OArchiver> : mpl::true_ { }; \
100 \
101 template<> \
102 struct output_archiver< IArchiver > { typedef OArchiver type; }; \
103 \
104 template<> \
105 struct input_archiver< OArchiver > { typedef IArchiver type; }; \
106 } }
107
108 /**
109 * @brief Define the implementation for Boost.Serialization archivers
110 * that can be used for Python objects and C++ objects wrapped in
111 * Python.
112 *
113 * This macro can only be expanded from the global namespace. It only
114 * requires that IArchiver and OArchiver be forward-declared. Before
115 * using this macro, you will need to declare IArchiver and OArchiver
116 * as direct serialization archives with
117 * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(IArchiver, OArchiver).
118 */
119 #define BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL(IArchiver, OArchiver) \
120 namespace boost { namespace python { namespace detail { \
121 template \
122 class BOOST_MPI_PYTHON_DECL direct_serialization_table< IArchiver , OArchiver >; \
123 \
124 template<> \
125 BOOST_MPI_PYTHON_DECL \
126 direct_serialization_table< IArchiver , OArchiver >& \
127 get_direct_serialization_table< IArchiver , OArchiver >( ) \
128 { \
129 static direct_serialization_table< IArchiver, OArchiver > table; \
130 return table; \
131 } \
132 } } }
133
134 namespace boost { namespace python {
135
136 /**
137 * INTERNAL ONLY
138 *
139 * Provides access to the Python "pickle" module from within C++.
140 */
141 class BOOST_MPI_PYTHON_DECL pickle {
142 struct data_t;
143
144 public:
145 static object dumps(object obj, int protocol = -1);
146 static object loads(object s);
147
148 private:
149 static void initialize_data();
150
151 static data_t* data;
152 };
153
154 /**
155 * @brief Whether the input/output archiver pair has "direct"
156 * serialization for C++ objects exposed in Python.
157 *
158 * Users do not typically need to specialize this trait, as it will be
159 * specialized as part of the macro
160 * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
161 */
162 template<typename IArchiver, typename OArchiver>
163 struct has_direct_serialization : mpl::false_ { };
164
165 /**
166 * @brief A metafunction that determines the output archiver for the
167 * given input archiver.
168 *
169 * Users do not typically need to specialize this trait, as it will be
170 * specialized as part of the macro
171 * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
172 */
173 template<typename IArchiver> struct output_archiver { };
174
175 /**
176 * @brief A metafunction that determines the input archiver for the
177 * given output archiver.
178 *
179 * Users do not typically need to specialize this trait, as it will be
180 * specialized as part of the macro
181 * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE.
182 *
183 */
184 template<typename OArchiver> struct input_archiver { };
185
186 namespace detail {
187
188 /**
189 * INTERNAL ONLY
190 *
191 * This class contains the direct-serialization code for the given
192 * IArchiver/OArchiver pair. It is intended to be used as a
193 * singleton class, and will be accessed when (de-)serializing a
194 * Boost.Python object with an archiver that supports direct
195 * serializations. Do not create instances of this class directly:
196 * instead, use get_direct_serialization_table.
197 */
198 template<typename IArchiver, typename OArchiver>
199 class BOOST_MPI_PYTHON_DECL direct_serialization_table
200 {
201 public:
202 typedef boost::function3<void, OArchiver&, const object&, const unsigned int>
203 saver_t;
204 typedef boost::function3<void, IArchiver&, object&, const unsigned int>
205 loader_t;
206
207 typedef std::map<PyTypeObject*, std::pair<int, saver_t> > savers_t;
208 typedef std::map<int, loader_t> loaders_t;
209
210 /**
211 * Retrieve the saver (serializer) associated with the Python
212 * object @p obj.
213 *
214 * @param obj The object we want to save. Only its (Python) type
215 * is important.
216 *
217 * @param descriptor The value of the descriptor associated to
218 * the returned saver. Will be set to zero if no saver was found
219 * for @p obj.
220 *
221 * @returns a function object that can be used to serialize this
222 * object (and other objects of the same type), if possible. If
223 * no saver can be found, returns an empty function object..
224 */
saver(const object & obj,int & descriptor)225 saver_t saver(const object& obj, int& descriptor)
226 {
227 typename savers_t::iterator pos = savers.find(obj.ptr()->ob_type);
228 if (pos != savers.end()) {
229 descriptor = pos->second.first;
230 return pos->second.second;
231 }
232 else {
233 descriptor = 0;
234 return saver_t();
235 }
236 }
237
238 /**
239 * Retrieve the loader (deserializer) associated with the given
240 * descriptor.
241 *
242 * @param descriptor The descriptor number provided by saver()
243 * when determining the saver for this type.
244 *
245 * @returns a function object that can be used to deserialize an
246 * object whose type is the same as that corresponding to the
247 * descriptor. If the descriptor is unknown, the return value
248 * will be an empty function object.
249 */
loader(int descriptor)250 loader_t loader(int descriptor)
251 {
252 typename loaders_t::iterator pos = loaders.find(descriptor);
253 if (pos != loaders.end())
254 return pos->second;
255 else
256 return loader_t();
257 }
258
259 /**
260 * Register the type T for direct serialization.
261 *
262 * @param value A sample value of the type @c T. This may be used
263 * to compute the Python type associated with the C++ type @c T.
264 *
265 * @param type The Python type associated with the C++ type @c
266 * T. If not provided, it will be computed from the same value @p
267 * value.
268 */
269 template<typename T>
register_type(const T & value=T (),PyTypeObject * type=0)270 void register_type(const T& value = T(), PyTypeObject* type = 0)
271 {
272 // If the user did not provide us with a Python type, figure it
273 // out for ourselves.
274 if (!type) {
275 object obj(value);
276 type = obj.ptr()->ob_type;
277 }
278
279 register_type(default_saver<T>(), default_loader<T>(type), value, type);
280 }
281
282 /**
283 * Register the type T for direct serialization.
284 *
285 * @param saver A function object that will serialize a
286 * Boost.Python object (that represents a C++ object of type @c
287 * T) to an @c OArchive.
288 *
289 * @param loader A function object that will deserialize from an
290 * @c IArchive into a Boost.Python object that represents a C++
291 * object of type @c T.
292 *
293 * @param value A sample value of the type @c T. This may be used
294 * to compute the Python type associated with the C++ type @c T.
295 *
296 * @param type The Python type associated with the C++ type @c
297 * T. If not provided, it will be computed from the same value @p
298 * value.
299 */
300 template<typename T>
register_type(const saver_t & saver,const loader_t & loader,const T & value=T (),PyTypeObject * type=0)301 void register_type(const saver_t& saver, const loader_t& loader,
302 const T& value = T(), PyTypeObject* type = 0)
303 {
304 // If the user did not provide us with a Python type, figure it
305 // out for ourselves.
306 if (!type) {
307 object obj(value);
308 type = obj.ptr()->ob_type;
309 }
310
311 int descriptor = savers.size() + 1;
312 if (savers.find(type) != savers.end())
313 return;
314
315 savers[type] = std::make_pair(descriptor, saver);
316 loaders[descriptor] = loader;
317 }
318
319 protected:
320 template<typename T>
321 struct default_saver {
operator ()boost::python::detail::direct_serialization_table::default_saver322 void operator()(OArchiver& ar, const object& obj, const unsigned int) {
323 T value = extract<T>(obj)();
324 ar << value;
325 }
326 };
327
328 template<typename T>
329 struct default_loader {
default_loaderboost::python::detail::direct_serialization_table::default_loader330 default_loader(PyTypeObject* type) : type(type) { }
331
operator ()boost::python::detail::direct_serialization_table::default_loader332 void operator()(IArchiver& ar, object& obj, const unsigned int) {
333 // If we can, extract the object in place.
334 if (!is_fundamental<T>::value && obj && obj.ptr()->ob_type == type) {
335 ar >> extract<T&>(obj)();
336 } else {
337 T value;
338 ar >> value;
339 obj = object(value);
340 }
341 }
342
343 private:
344 PyTypeObject* type;
345 };
346
347 savers_t savers;
348 loaders_t loaders;
349 };
350
351 /**
352 * @brief Retrieve the direct-serialization table for an
353 * IArchiver/OArchiver pair.
354 *
355 * This function is responsible for returning a reference to the
356 * singleton direct-serialization table. Its primary template is
357 * left undefined, to force the use of an explicit specialization
358 * with a definition in a single translation unit. Use the macro
359 * BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE_IMPL to define this
360 * explicit specialization.
361 */
362 template<typename IArchiver, typename OArchiver>
363 direct_serialization_table<IArchiver, OArchiver>&
364 get_direct_serialization_table();
365 } // end namespace detail
366
367 /**
368 * @brief Register the type T for direct serialization.
369 *
370 * The @c register_serialized function registers a C++ type for direct
371 * serialization with the given @c IArchiver/@c OArchiver pair. Direct
372 * serialization elides the use of the Python @c pickle package when
373 * serializing Python objects that represent C++ values. Direct
374 * serialization can be beneficial both to improve serialization
375 * performance (Python pickling can be very inefficient) and to permit
376 * serialization for Python-wrapped C++ objects that do not support
377 * pickling.
378 *
379 * @param value A sample value of the type @c T. This may be used
380 * to compute the Python type associated with the C++ type @c T.
381 *
382 * @param type The Python type associated with the C++ type @c
383 * T. If not provided, it will be computed from the same value @p
384 * value.
385 */
386 template<typename IArchiver, typename OArchiver, typename T>
387 void
register_serialized(const T & value=T (),PyTypeObject * type=0)388 register_serialized(const T& value = T(), PyTypeObject* type = 0)
389 {
390 detail::direct_serialization_table<IArchiver, OArchiver>& table =
391 detail::get_direct_serialization_table<IArchiver, OArchiver>();
392 table.register_type(value, type);
393 }
394
395 namespace detail {
396
397 /// Save a Python object by pickling it.
398 template<typename Archiver>
399 void
save_impl(Archiver & ar,const boost::python::object & obj,const unsigned int,mpl::false_)400 save_impl(Archiver& ar, const boost::python::object& obj,
401 const unsigned int /*version*/,
402 mpl::false_ /*has_direct_serialization*/)
403 {
404 boost::python::object bytes = boost::python::pickle::dumps(obj);
405 int sz = PyBytes_Size(bytes.ptr());
406 char *data = PyBytes_AsString(bytes.ptr());
407 ar << sz << boost::serialization::make_array(data, sz);
408 }
409
410 /// Try to save a Python object by directly serializing it; fall back
411 /// on pickling if required.
412 template<typename Archiver>
413 void
save_impl(Archiver & ar,const boost::python::object & obj,const unsigned int version,mpl::true_)414 save_impl(Archiver& ar, const boost::python::object& obj,
415 const unsigned int version,
416 mpl::true_ /*has_direct_serialization*/)
417 {
418 typedef Archiver OArchiver;
419 typedef typename input_archiver<OArchiver>::type IArchiver;
420 typedef typename direct_serialization_table<IArchiver, OArchiver>::saver_t
421 saver_t;
422
423 direct_serialization_table<IArchiver, OArchiver>& table =
424 get_direct_serialization_table<IArchiver, OArchiver>();
425
426 int descriptor = 0;
427 if (saver_t saver = table.saver(obj, descriptor)) {
428 ar << descriptor;
429 saver(ar, obj, version);
430 } else {
431 // Pickle it
432 ar << descriptor;
433 detail::save_impl(ar, obj, version, mpl::false_());
434 }
435 }
436
437 /// Load a Python object by unpickling it
438 template<typename Archiver>
439 void
load_impl(Archiver & ar,boost::python::object & obj,const unsigned int,mpl::false_)440 load_impl(Archiver& ar, boost::python::object& obj,
441 const unsigned int /*version*/,
442 mpl::false_ /*has_direct_serialization*/)
443 {
444 int len;
445 ar >> len;
446 boost::scoped_array<char> data(new char[len]);
447 ar >> boost::serialization::make_array(data.get(), len);
448 boost::python::object bytes(boost::python::handle<>(PyBytes_FromStringAndSize(data.get(), len)));
449 obj = boost::python::pickle::loads(bytes);
450 }
451
452 /// Try to load a Python object by directly deserializing it; fall back
453 /// on unpickling if required.
454 template<typename Archiver>
455 void
load_impl(Archiver & ar,boost::python::object & obj,const unsigned int version,mpl::true_)456 load_impl(Archiver& ar, boost::python::object& obj,
457 const unsigned int version,
458 mpl::true_ /*has_direct_serialization*/)
459 {
460 typedef Archiver IArchiver;
461 typedef typename output_archiver<IArchiver>::type OArchiver;
462 typedef typename direct_serialization_table<IArchiver, OArchiver>::loader_t
463 loader_t;
464
465 direct_serialization_table<IArchiver, OArchiver>& table =
466 get_direct_serialization_table<IArchiver, OArchiver>();
467
468 int descriptor;
469 ar >> descriptor;
470
471 if (descriptor) {
472 loader_t loader = table.loader(descriptor);
473 BOOST_ASSERT(loader);
474
475 loader(ar, obj, version);
476 } else {
477 // Unpickle it
478 detail::load_impl(ar, obj, version, mpl::false_());
479 }
480 }
481
482 } // end namespace detail
483
484 template<typename Archiver>
485 void
save(Archiver & ar,const boost::python::object & obj,const unsigned int version)486 save(Archiver& ar, const boost::python::object& obj,
487 const unsigned int version)
488 {
489 typedef Archiver OArchiver;
490 typedef typename input_archiver<OArchiver>::type IArchiver;
491 detail::save_impl(ar, obj, version,
492 has_direct_serialization<IArchiver, OArchiver>());
493 }
494
495 template<typename Archiver>
496 void
load(Archiver & ar,boost::python::object & obj,const unsigned int version)497 load(Archiver& ar, boost::python::object& obj,
498 const unsigned int version)
499 {
500 typedef Archiver IArchiver;
501 typedef typename output_archiver<IArchiver>::type OArchiver;
502 detail::load_impl(ar, obj, version,
503 has_direct_serialization<IArchiver, OArchiver>());
504 }
505
506 template<typename Archive>
507 inline void
serialize(Archive & ar,boost::python::object & obj,const unsigned int version)508 serialize(Archive& ar, boost::python::object& obj, const unsigned int version)
509 {
510 boost::serialization::split_free(ar, obj, version);
511 }
512
513 } } // end namespace boost::python
514
515 /************************************************************************
516 * Boost.MPI-Specific Section *
517 ************************************************************************/
518 namespace boost { namespace mpi {
519 class packed_iarchive;
520 class packed_oarchive;
521 } } // end namespace boost::mpi
522
523 BOOST_PYTHON_DIRECT_SERIALIZATION_ARCHIVE(
524 ::boost::mpi::packed_iarchive,
525 ::boost::mpi::packed_oarchive)
526
527 namespace boost { namespace mpi { namespace python {
528
529 template<typename T>
530 void
register_serialized(const T & value,PyTypeObject * type)531 register_serialized(const T& value, PyTypeObject* type)
532 {
533 using boost::python::register_serialized;
534 register_serialized<packed_iarchive, packed_oarchive>(value, type);
535 }
536
537 } } } // end namespace boost::mpi::python
538
539 #endif // BOOST_MPI_PYTHON_SERIALIZE_HPP
540