1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 //
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file or at
6 // https://developers.google.com/open-source/licenses/bsd
7
8 // Author: anuraag@google.com (Anuraag Agrawal)
9 // Author: tibell@google.com (Johan Tibell)
10
11 #include "google/protobuf/pyext/repeated_composite_container.h"
12
13 #include <memory>
14
15 #include "google/protobuf/descriptor.h"
16 #include "google/protobuf/dynamic_message.h"
17 #include "google/protobuf/message.h"
18 #include "google/protobuf/reflection.h"
19 #include "google/protobuf/pyext/descriptor.h"
20 #include "google/protobuf/pyext/descriptor_pool.h"
21 #include "google/protobuf/pyext/message.h"
22 #include "google/protobuf/pyext/message_factory.h"
23 #include "google/protobuf/pyext/scoped_pyobject_ptr.h"
24
25 namespace google {
26 namespace protobuf {
27 namespace python {
28
29 namespace repeated_composite_container {
30
31 // ---------------------------------------------------------------------
32 // len()
33
Length(PyObject * pself)34 static Py_ssize_t Length(PyObject* pself) {
35 RepeatedCompositeContainer* self =
36 reinterpret_cast<RepeatedCompositeContainer*>(pself);
37
38 Message* message = self->parent->message;
39 return message->GetReflection()->FieldSize(*message,
40 self->parent_field_descriptor);
41 }
42
43 // ---------------------------------------------------------------------
44 // add()
45
Add(RepeatedCompositeContainer * self,PyObject * args,PyObject * kwargs)46 PyObject* Add(RepeatedCompositeContainer* self, PyObject* args,
47 PyObject* kwargs) {
48 if (cmessage::AssureWritable(self->parent) == -1) return nullptr;
49 Message* message = self->parent->message;
50
51 Message* sub_message = message->GetReflection()->AddMessage(
52 message, self->parent_field_descriptor,
53 self->child_message_class->py_message_factory->message_factory);
54 CMessage* cmsg = self->parent->BuildSubMessageFromPointer(
55 self->parent_field_descriptor, sub_message, self->child_message_class);
56
57 if (cmessage::InitAttributes(cmsg, args, kwargs) < 0) {
58 message->GetReflection()->RemoveLast(message,
59 self->parent_field_descriptor);
60 Py_DECREF(cmsg);
61 return nullptr;
62 }
63
64 return cmsg->AsPyObject();
65 }
66
AddMethod(PyObject * self,PyObject * args,PyObject * kwargs)67 static PyObject* AddMethod(PyObject* self, PyObject* args, PyObject* kwargs) {
68 return Add(reinterpret_cast<RepeatedCompositeContainer*>(self), args, kwargs);
69 }
70
71 // ---------------------------------------------------------------------
72 // append()
73
AddMessage(RepeatedCompositeContainer * self,PyObject * value)74 static PyObject* AddMessage(RepeatedCompositeContainer* self, PyObject* value) {
75 cmessage::AssureWritable(self->parent);
76 PyObject* py_cmsg;
77 Message* message = self->parent->message;
78 const Reflection* reflection = message->GetReflection();
79 py_cmsg = Add(self, nullptr, nullptr);
80 if (py_cmsg == nullptr) return nullptr;
81 CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
82 if (ScopedPyObjectPtr(cmessage::MergeFrom(cmsg, value)) == nullptr) {
83 reflection->RemoveLast(message, self->parent_field_descriptor);
84 Py_DECREF(cmsg);
85 return nullptr;
86 }
87 return py_cmsg;
88 }
89
AppendMethod(PyObject * pself,PyObject * value)90 static PyObject* AppendMethod(PyObject* pself, PyObject* value) {
91 RepeatedCompositeContainer* self =
92 reinterpret_cast<RepeatedCompositeContainer*>(pself);
93 ScopedPyObjectPtr py_cmsg(AddMessage(self, value));
94 if (py_cmsg == nullptr) {
95 return nullptr;
96 }
97
98 Py_RETURN_NONE;
99 }
100
101 // ---------------------------------------------------------------------
102 // insert()
Insert(PyObject * pself,PyObject * args)103 static PyObject* Insert(PyObject* pself, PyObject* args) {
104 RepeatedCompositeContainer* self =
105 reinterpret_cast<RepeatedCompositeContainer*>(pself);
106
107 Py_ssize_t index;
108 PyObject* value;
109 if (!PyArg_ParseTuple(args, "nO", &index, &value)) {
110 return nullptr;
111 }
112
113 ScopedPyObjectPtr py_cmsg(AddMessage(self, value));
114 if (py_cmsg == nullptr) {
115 return nullptr;
116 }
117
118 // Swap the element to right position.
119 Message* message = self->parent->message;
120 const Reflection* reflection = message->GetReflection();
121 const FieldDescriptor* field_descriptor = self->parent_field_descriptor;
122 Py_ssize_t length = reflection->FieldSize(*message, field_descriptor) - 1;
123 Py_ssize_t end_index = index;
124 if (end_index < 0) end_index += length;
125 if (end_index < 0) end_index = 0;
126 for (Py_ssize_t i = length; i > end_index; i--) {
127 reflection->SwapElements(message, field_descriptor, i, i - 1);
128 }
129
130 Py_RETURN_NONE;
131 }
132
133 // ---------------------------------------------------------------------
134 // extend()
135
Extend(RepeatedCompositeContainer * self,PyObject * value)136 PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value) {
137 cmessage::AssureWritable(self->parent);
138 ScopedPyObjectPtr iter(PyObject_GetIter(value));
139 if (iter == nullptr) {
140 PyErr_SetString(PyExc_TypeError, "Value must be iterable");
141 return nullptr;
142 }
143 ScopedPyObjectPtr next;
144 while ((next.reset(PyIter_Next(iter.get()))) != nullptr) {
145 if (!PyObject_TypeCheck(next.get(), CMessage_Type)) {
146 PyErr_SetString(PyExc_TypeError, "Not a cmessage");
147 return nullptr;
148 }
149 ScopedPyObjectPtr new_message(Add(self, nullptr, nullptr));
150 if (new_message == nullptr) {
151 return nullptr;
152 }
153 CMessage* new_cmessage = reinterpret_cast<CMessage*>(new_message.get());
154 if (ScopedPyObjectPtr(cmessage::MergeFrom(new_cmessage, next.get())) ==
155 nullptr) {
156 return nullptr;
157 }
158 }
159 if (PyErr_Occurred()) {
160 return nullptr;
161 }
162 Py_RETURN_NONE;
163 }
164
ExtendMethod(PyObject * self,PyObject * value)165 static PyObject* ExtendMethod(PyObject* self, PyObject* value) {
166 return Extend(reinterpret_cast<RepeatedCompositeContainer*>(self), value);
167 }
168
MergeFrom(RepeatedCompositeContainer * self,PyObject * other)169 PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other) {
170 return Extend(self, other);
171 }
172
MergeFromMethod(PyObject * self,PyObject * other)173 static PyObject* MergeFromMethod(PyObject* self, PyObject* other) {
174 return MergeFrom(reinterpret_cast<RepeatedCompositeContainer*>(self), other);
175 }
176
177 // This function does not check the bounds.
GetItem(RepeatedCompositeContainer * self,Py_ssize_t index,Py_ssize_t length=-1)178 static PyObject* GetItem(RepeatedCompositeContainer* self, Py_ssize_t index,
179 Py_ssize_t length = -1) {
180 if (length == -1) {
181 Message* message = self->parent->message;
182 const Reflection* reflection = message->GetReflection();
183 length = reflection->FieldSize(*message, self->parent_field_descriptor);
184 }
185 if (index < 0 || index >= length) {
186 PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index);
187 return nullptr;
188 }
189 Message* message = self->parent->message;
190 Message* sub_message = message->GetReflection()->MutableRepeatedMessage(
191 message, self->parent_field_descriptor, index);
192 return self->parent
193 ->BuildSubMessageFromPointer(self->parent_field_descriptor, sub_message,
194 self->child_message_class)
195 ->AsPyObject();
196 }
197
Subscript(RepeatedCompositeContainer * self,PyObject * item)198 PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* item) {
199 Message* message = self->parent->message;
200 const Reflection* reflection = message->GetReflection();
201 Py_ssize_t length =
202 reflection->FieldSize(*message, self->parent_field_descriptor);
203
204 if (PyIndex_Check(item)) {
205 Py_ssize_t index;
206 index = PyNumber_AsSsize_t(item, PyExc_IndexError);
207 if (index == -1 && PyErr_Occurred()) return nullptr;
208 if (index < 0) index += length;
209 return GetItem(self, index, length);
210 } else if (PySlice_Check(item)) {
211 Py_ssize_t from, to, step, slicelength, cur, i;
212 PyObject* result;
213
214 if (PySlice_GetIndicesEx(item, length, &from, &to, &step, &slicelength) ==
215 -1) {
216 return nullptr;
217 }
218
219 if (slicelength <= 0) {
220 return PyList_New(0);
221 } else {
222 result = PyList_New(slicelength);
223 if (!result) return nullptr;
224
225 for (cur = from, i = 0; i < slicelength; cur += step, i++) {
226 PyList_SET_ITEM(result, i, GetItem(self, cur, length));
227 }
228
229 return result;
230 }
231 } else {
232 PyErr_Format(PyExc_TypeError, "indices must be integers, not %.200s",
233 item->ob_type->tp_name);
234 return nullptr;
235 }
236 }
237
SubscriptMethod(PyObject * self,PyObject * slice)238 static PyObject* SubscriptMethod(PyObject* self, PyObject* slice) {
239 return Subscript(reinterpret_cast<RepeatedCompositeContainer*>(self), slice);
240 }
241
AssignSubscript(RepeatedCompositeContainer * self,PyObject * slice,PyObject * value)242 int AssignSubscript(RepeatedCompositeContainer* self, PyObject* slice,
243 PyObject* value) {
244 if (value != nullptr) {
245 PyErr_SetString(PyExc_TypeError, "does not support assignment");
246 return -1;
247 }
248
249 return cmessage::DeleteRepeatedField(self->parent,
250 self->parent_field_descriptor, slice);
251 }
252
AssignSubscriptMethod(PyObject * self,PyObject * slice,PyObject * value)253 static int AssignSubscriptMethod(PyObject* self, PyObject* slice,
254 PyObject* value) {
255 return AssignSubscript(reinterpret_cast<RepeatedCompositeContainer*>(self),
256 slice, value);
257 }
258
Remove(PyObject * pself,PyObject * value)259 static PyObject* Remove(PyObject* pself, PyObject* value) {
260 RepeatedCompositeContainer* self =
261 reinterpret_cast<RepeatedCompositeContainer*>(pself);
262 Py_ssize_t len = Length(reinterpret_cast<PyObject*>(self));
263
264 for (Py_ssize_t i = 0; i < len; i++) {
265 ScopedPyObjectPtr item(GetItem(self, i, len));
266 if (item == nullptr) {
267 return nullptr;
268 }
269 int result = PyObject_RichCompareBool(item.get(), value, Py_EQ);
270 if (result < 0) {
271 return nullptr;
272 }
273 if (result) {
274 ScopedPyObjectPtr py_index(PyLong_FromSsize_t(i));
275 if (AssignSubscript(self, py_index.get(), nullptr) < 0) {
276 return nullptr;
277 }
278 Py_RETURN_NONE;
279 }
280 }
281 PyErr_SetString(PyExc_ValueError, "Item to delete not in list");
282 return nullptr;
283 }
284
RichCompare(PyObject * pself,PyObject * other,int opid)285 static PyObject* RichCompare(PyObject* pself, PyObject* other, int opid) {
286 RepeatedCompositeContainer* self =
287 reinterpret_cast<RepeatedCompositeContainer*>(pself);
288
289 if (!PyObject_TypeCheck(other, &RepeatedCompositeContainer_Type)) {
290 PyErr_SetString(PyExc_TypeError,
291 "Can only compare repeated composite fields "
292 "against other repeated composite fields.");
293 return nullptr;
294 }
295 if (opid == Py_EQ || opid == Py_NE) {
296 // TODO: Don't make new lists just for this...
297 ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
298 if (full_slice == nullptr) {
299 return nullptr;
300 }
301 ScopedPyObjectPtr list(Subscript(self, full_slice.get()));
302 if (list == nullptr) {
303 return nullptr;
304 }
305 ScopedPyObjectPtr other_list(
306 Subscript(reinterpret_cast<RepeatedCompositeContainer*>(other),
307 full_slice.get()));
308 if (other_list == nullptr) {
309 return nullptr;
310 }
311 return PyObject_RichCompare(list.get(), other_list.get(), opid);
312 } else {
313 Py_INCREF(Py_NotImplemented);
314 return Py_NotImplemented;
315 }
316 }
317
ToStr(PyObject * pself)318 static PyObject* ToStr(PyObject* pself) {
319 ScopedPyObjectPtr full_slice(PySlice_New(nullptr, nullptr, nullptr));
320 if (full_slice == nullptr) {
321 return nullptr;
322 }
323 ScopedPyObjectPtr list(Subscript(
324 reinterpret_cast<RepeatedCompositeContainer*>(pself), full_slice.get()));
325 if (list == nullptr) {
326 return nullptr;
327 }
328 return PyObject_Repr(list.get());
329 }
330
331 // ---------------------------------------------------------------------
332 // sort()
333
ReorderAttached(RepeatedCompositeContainer * self,PyObject * child_list)334 static void ReorderAttached(RepeatedCompositeContainer* self,
335 PyObject* child_list) {
336 Message* message = self->parent->message;
337 const Reflection* reflection = message->GetReflection();
338 const FieldDescriptor* descriptor = self->parent_field_descriptor;
339 const Py_ssize_t length = Length(reinterpret_cast<PyObject*>(self));
340
341 // We need to rearrange things to match python's sort order.
342 for (Py_ssize_t i = 0; i < length; ++i) {
343 reflection->UnsafeArenaReleaseLast(message, descriptor);
344 }
345 for (Py_ssize_t i = 0; i < length; ++i) {
346 Message* child_message =
347 reinterpret_cast<CMessage*>(PyList_GET_ITEM(child_list, i))->message;
348 reflection->UnsafeArenaAddAllocatedMessage(message, descriptor,
349 child_message);
350 }
351 }
352
353 // Returns 0 if successful; returns -1 and sets an exception if
354 // unsuccessful.
SortPythonMessages(RepeatedCompositeContainer * self,PyObject * args,PyObject * kwds)355 static int SortPythonMessages(RepeatedCompositeContainer* self, PyObject* args,
356 PyObject* kwds) {
357 ScopedPyObjectPtr child_list(
358 PySequence_List(reinterpret_cast<PyObject*>(self)));
359 if (child_list == nullptr) {
360 return -1;
361 }
362 ScopedPyObjectPtr m(PyObject_GetAttrString(child_list.get(), "sort"));
363 if (m == nullptr) return -1;
364 if (ScopedPyObjectPtr(PyObject_Call(m.get(), args, kwds)) == nullptr)
365 return -1;
366 ReorderAttached(self, child_list.get());
367 return 0;
368 }
369
Sort(PyObject * pself,PyObject * args,PyObject * kwds)370 static PyObject* Sort(PyObject* pself, PyObject* args, PyObject* kwds) {
371 RepeatedCompositeContainer* self =
372 reinterpret_cast<RepeatedCompositeContainer*>(pself);
373
374 // Support the old sort_function argument for backwards
375 // compatibility.
376 if (kwds != nullptr) {
377 PyObject* sort_func = PyDict_GetItemString(kwds, "sort_function");
378 if (sort_func != nullptr) {
379 // Must set before deleting as sort_func is a borrowed reference
380 // and kwds might be the only thing keeping it alive.
381 PyDict_SetItemString(kwds, "cmp", sort_func);
382 PyDict_DelItemString(kwds, "sort_function");
383 }
384 }
385
386 if (SortPythonMessages(self, args, kwds) < 0) {
387 return nullptr;
388 }
389 Py_RETURN_NONE;
390 }
391
392 // ---------------------------------------------------------------------
393 // reverse()
394
395 // Returns 0 if successful; returns -1 and sets an exception if
396 // unsuccessful.
ReversePythonMessages(RepeatedCompositeContainer * self)397 static int ReversePythonMessages(RepeatedCompositeContainer* self) {
398 ScopedPyObjectPtr child_list(
399 PySequence_List(reinterpret_cast<PyObject*>(self)));
400 if (child_list == nullptr) {
401 return -1;
402 }
403 if (ScopedPyObjectPtr(
404 PyObject_CallMethod(child_list.get(), "reverse", nullptr)) == nullptr)
405 return -1;
406 ReorderAttached(self, child_list.get());
407 return 0;
408 }
409
Reverse(PyObject * pself)410 static PyObject* Reverse(PyObject* pself) {
411 RepeatedCompositeContainer* self =
412 reinterpret_cast<RepeatedCompositeContainer*>(pself);
413
414 if (ReversePythonMessages(self) < 0) {
415 return nullptr;
416 }
417 Py_RETURN_NONE;
418 }
419
420 // ---------------------------------------------------------------------
421
Item(PyObject * pself,Py_ssize_t index)422 static PyObject* Item(PyObject* pself, Py_ssize_t index) {
423 RepeatedCompositeContainer* self =
424 reinterpret_cast<RepeatedCompositeContainer*>(pself);
425 return GetItem(self, index);
426 }
427
Pop(PyObject * pself,PyObject * args)428 static PyObject* Pop(PyObject* pself, PyObject* args) {
429 RepeatedCompositeContainer* self =
430 reinterpret_cast<RepeatedCompositeContainer*>(pself);
431
432 Py_ssize_t index = -1;
433 if (!PyArg_ParseTuple(args, "|n", &index)) {
434 return nullptr;
435 }
436 Py_ssize_t length = Length(pself);
437 if (index < 0) index += length;
438 PyObject* item = GetItem(self, index, length);
439 if (item == nullptr) {
440 return nullptr;
441 }
442 ScopedPyObjectPtr py_index(PyLong_FromSsize_t(index));
443 if (AssignSubscript(self, py_index.get(), nullptr) < 0) {
444 return nullptr;
445 }
446 return item;
447 }
448
DeepCopy(PyObject * pself,PyObject * arg)449 PyObject* DeepCopy(PyObject* pself, PyObject* arg) {
450 return reinterpret_cast<RepeatedCompositeContainer*>(pself)->DeepCopy();
451 }
452
453 // The private constructor of RepeatedCompositeContainer objects.
NewContainer(CMessage * parent,const FieldDescriptor * parent_field_descriptor,CMessageClass * child_message_class)454 RepeatedCompositeContainer* NewContainer(
455 CMessage* parent, const FieldDescriptor* parent_field_descriptor,
456 CMessageClass* child_message_class) {
457 if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) {
458 return nullptr;
459 }
460
461 RepeatedCompositeContainer* self =
462 reinterpret_cast<RepeatedCompositeContainer*>(
463 PyType_GenericAlloc(&RepeatedCompositeContainer_Type, 0));
464 if (self == nullptr) {
465 return nullptr;
466 }
467
468 Py_INCREF(parent);
469 self->parent = parent;
470 self->parent_field_descriptor = parent_field_descriptor;
471 Py_INCREF(child_message_class);
472 self->child_message_class = child_message_class;
473 return self;
474 }
475
Dealloc(PyObject * pself)476 static void Dealloc(PyObject* pself) {
477 RepeatedCompositeContainer* self =
478 reinterpret_cast<RepeatedCompositeContainer*>(pself);
479 self->RemoveFromParentCache();
480 Py_CLEAR(self->child_message_class);
481 Py_TYPE(self)->tp_free(pself);
482 }
483
484 static PySequenceMethods SqMethods = {
485 Length, /* sq_length */
486 nullptr, /* sq_concat */
487 nullptr, /* sq_repeat */
488 Item /* sq_item */
489 };
490
491 static PyMappingMethods MpMethods = {
492 Length, /* mp_length */
493 SubscriptMethod, /* mp_subscript */
494 AssignSubscriptMethod, /* mp_ass_subscript */
495 };
496
497 static PyMethodDef Methods[] = {
498 {"__deepcopy__", DeepCopy, METH_VARARGS, "Makes a deep copy of the class."},
499 {"add", reinterpret_cast<PyCFunction>(AddMethod),
500 METH_VARARGS | METH_KEYWORDS, "Adds an object to the repeated container."},
501 {"append", AppendMethod, METH_O,
502 "Appends a message to the end of the repeated container."},
503 {"insert", Insert, METH_VARARGS,
504 "Inserts a message before the specified index."},
505 {"extend", ExtendMethod, METH_O, "Adds objects to the repeated container."},
506 {"pop", Pop, METH_VARARGS,
507 "Removes an object from the repeated container and returns it."},
508 {"remove", Remove, METH_O,
509 "Removes an object from the repeated container."},
510 {"sort", reinterpret_cast<PyCFunction>(Sort), METH_VARARGS | METH_KEYWORDS,
511 "Sorts the repeated container."},
512 {"reverse", reinterpret_cast<PyCFunction>(Reverse), METH_NOARGS,
513 "Reverses elements order of the repeated container."},
514 {"MergeFrom", MergeFromMethod, METH_O,
515 "Adds objects to the repeated container."},
516 {nullptr, nullptr}};
517
518 } // namespace repeated_composite_container
519
520 PyTypeObject RepeatedCompositeContainer_Type = {
521 PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
522 ".RepeatedCompositeContainer", // tp_name
523 sizeof(RepeatedCompositeContainer), // tp_basicsize
524 0, // tp_itemsize
525 repeated_composite_container::Dealloc, // tp_dealloc
526 #if PY_VERSION_HEX >= 0x03080000
527 0, // tp_vectorcall_offset
528 #else
529 nullptr, // tp_print
530 #endif
531 nullptr, // tp_getattr
532 nullptr, // tp_setattr
533 nullptr, // tp_compare
534 repeated_composite_container::ToStr, // tp_repr
535 nullptr, // tp_as_number
536 &repeated_composite_container::SqMethods, // tp_as_sequence
537 &repeated_composite_container::MpMethods, // tp_as_mapping
538 PyObject_HashNotImplemented, // tp_hash
539 nullptr, // tp_call
540 nullptr, // tp_str
541 nullptr, // tp_getattro
542 nullptr, // tp_setattro
543 nullptr, // tp_as_buffer
544 #if PY_VERSION_HEX >= 0x030A0000
545 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_SEQUENCE, // tp_flags
546 #else
547 Py_TPFLAGS_DEFAULT, // tp_flags
548 #endif
549 "A Repeated scalar container", // tp_doc
550 nullptr, // tp_traverse
551 nullptr, // tp_clear
552 repeated_composite_container::RichCompare, // tp_richcompare
553 0, // tp_weaklistoffset
554 nullptr, // tp_iter
555 nullptr, // tp_iternext
556 repeated_composite_container::Methods, // tp_methods
557 nullptr, // tp_members
558 nullptr, // tp_getset
559 nullptr, // tp_base
560 nullptr, // tp_dict
561 nullptr, // tp_descr_get
562 nullptr, // tp_descr_set
563 0, // tp_dictoffset
564 nullptr, // tp_init
565 };
566
567 } // namespace python
568 } // namespace protobuf
569 } // namespace google
570