• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Implementation helper: a struct that looks like a tuple.
2    See timemodule and posixmodule for example uses.
3 
4    The structseq helper is considered an internal CPython implementation
5    detail.  Docs for modules using structseqs should call them
6    "named tuples" (be sure to include a space between the two
7    words and add a link back to the term in Docs/glossary.rst).
8 */
9 
10 #include "Python.h"
11 #include "pycore_tuple.h"         // _PyTuple_FromArray()
12 #include "pycore_object.h"        // _PyObject_GC_TRACK()
13 #include "structmember.h"         // PyMemberDef
14 #include "pycore_structseq.h"     // PyStructSequence_InitType()
15 
16 static const char visible_length_key[] = "n_sequence_fields";
17 static const char real_length_key[] = "n_fields";
18 static const char unnamed_fields_key[] = "n_unnamed_fields";
19 static const char match_args_key[] = "__match_args__";
20 
21 /* Fields with this name have only a field index, not a field name.
22    They are only allowed for indices < n_visible_fields. */
23 const char * const PyStructSequence_UnnamedField = "unnamed field";
24 
25 _Py_IDENTIFIER(n_sequence_fields);
26 _Py_IDENTIFIER(n_fields);
27 _Py_IDENTIFIER(n_unnamed_fields);
28 
29 static Py_ssize_t
get_type_attr_as_size(PyTypeObject * tp,_Py_Identifier * id)30 get_type_attr_as_size(PyTypeObject *tp, _Py_Identifier *id)
31 {
32     PyObject *name = _PyUnicode_FromId(id);
33     if (name == NULL) {
34         return -1;
35     }
36     PyObject *v = PyDict_GetItemWithError(tp->tp_dict, name);
37     if (v == NULL && !PyErr_Occurred()) {
38         PyErr_Format(PyExc_TypeError,
39                      "Missed attribute '%U' of type %s",
40                      name, tp->tp_name);
41     }
42     return PyLong_AsSsize_t(v);
43 }
44 
45 #define VISIBLE_SIZE(op) Py_SIZE(op)
46 #define VISIBLE_SIZE_TP(tp) get_type_attr_as_size(tp, &PyId_n_sequence_fields)
47 #define REAL_SIZE_TP(tp) get_type_attr_as_size(tp, &PyId_n_fields)
48 #define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op))
49 
50 #define UNNAMED_FIELDS_TP(tp) get_type_attr_as_size(tp, &PyId_n_unnamed_fields)
51 #define UNNAMED_FIELDS(op) UNNAMED_FIELDS_TP(Py_TYPE(op))
52 
53 
54 PyObject *
PyStructSequence_New(PyTypeObject * type)55 PyStructSequence_New(PyTypeObject *type)
56 {
57     PyStructSequence *obj;
58     Py_ssize_t size = REAL_SIZE_TP(type), i;
59     if (size < 0) {
60         return NULL;
61     }
62     Py_ssize_t vsize = VISIBLE_SIZE_TP(type);
63     if (vsize < 0) {
64         return NULL;
65     }
66 
67     obj = PyObject_GC_NewVar(PyStructSequence, type, size);
68     if (obj == NULL)
69         return NULL;
70     /* Hack the size of the variable object, so invisible fields don't appear
71      to Python code. */
72     Py_SET_SIZE(obj, vsize);
73     for (i = 0; i < size; i++)
74         obj->ob_item[i] = NULL;
75 
76     return (PyObject*)obj;
77 }
78 
79 void
PyStructSequence_SetItem(PyObject * op,Py_ssize_t i,PyObject * v)80 PyStructSequence_SetItem(PyObject* op, Py_ssize_t i, PyObject* v)
81 {
82     PyStructSequence_SET_ITEM(op, i, v);
83 }
84 
85 PyObject*
PyStructSequence_GetItem(PyObject * op,Py_ssize_t i)86 PyStructSequence_GetItem(PyObject* op, Py_ssize_t i)
87 {
88     return PyStructSequence_GET_ITEM(op, i);
89 }
90 
91 
92 static int
structseq_traverse(PyStructSequence * obj,visitproc visit,void * arg)93 structseq_traverse(PyStructSequence *obj, visitproc visit, void *arg)
94 {
95     if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HEAPTYPE) {
96         Py_VISIT(Py_TYPE(obj));
97     }
98     Py_ssize_t i, size;
99     size = REAL_SIZE(obj);
100     for (i = 0; i < size; ++i) {
101         Py_VISIT(obj->ob_item[i]);
102     }
103     return 0;
104 }
105 
106 static void
structseq_dealloc(PyStructSequence * obj)107 structseq_dealloc(PyStructSequence *obj)
108 {
109     Py_ssize_t i, size;
110     PyTypeObject *tp;
111     PyObject_GC_UnTrack(obj);
112 
113     tp = (PyTypeObject *) Py_TYPE(obj);
114     size = REAL_SIZE(obj);
115     for (i = 0; i < size; ++i) {
116         Py_XDECREF(obj->ob_item[i]);
117     }
118     PyObject_GC_Del(obj);
119     if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) {
120         Py_DECREF(tp);
121     }
122 }
123 
124 /*[clinic input]
125 class structseq "PyStructSequence *" "NULL"
126 [clinic start generated code]*/
127 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=9d781c6922c77752]*/
128 
129 #include "clinic/structseq.c.h"
130 
131 /*[clinic input]
132 @classmethod
133 structseq.__new__ as structseq_new
134     sequence as arg: object
135     dict: object(c_default="NULL") = {}
136 [clinic start generated code]*/
137 
138 static PyObject *
structseq_new_impl(PyTypeObject * type,PyObject * arg,PyObject * dict)139 structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict)
140 /*[clinic end generated code: output=baa082e788b171da input=90532511101aa3fb]*/
141 {
142     PyObject *ob;
143     PyStructSequence *res = NULL;
144     Py_ssize_t len, min_len, max_len, i, n_unnamed_fields;
145 
146     min_len = VISIBLE_SIZE_TP(type);
147     if (min_len < 0) {
148         return NULL;
149     }
150     max_len = REAL_SIZE_TP(type);
151     if (max_len < 0) {
152         return NULL;
153     }
154     n_unnamed_fields = UNNAMED_FIELDS_TP(type);
155     if (n_unnamed_fields < 0) {
156         return NULL;
157     }
158 
159     arg = PySequence_Fast(arg, "constructor requires a sequence");
160 
161     if (!arg) {
162         return NULL;
163     }
164 
165     if (dict && !PyDict_Check(dict)) {
166         PyErr_Format(PyExc_TypeError,
167                      "%.500s() takes a dict as second arg, if any",
168                      type->tp_name);
169         Py_DECREF(arg);
170         return NULL;
171     }
172 
173     len = PySequence_Fast_GET_SIZE(arg);
174     if (min_len != max_len) {
175         if (len < min_len) {
176             PyErr_Format(PyExc_TypeError,
177                 "%.500s() takes an at least %zd-sequence (%zd-sequence given)",
178                 type->tp_name, min_len, len);
179             Py_DECREF(arg);
180             return NULL;
181         }
182 
183         if (len > max_len) {
184             PyErr_Format(PyExc_TypeError,
185                 "%.500s() takes an at most %zd-sequence (%zd-sequence given)",
186                 type->tp_name, max_len, len);
187             Py_DECREF(arg);
188             return NULL;
189         }
190     }
191     else {
192         if (len != min_len) {
193             PyErr_Format(PyExc_TypeError,
194                          "%.500s() takes a %zd-sequence (%zd-sequence given)",
195                          type->tp_name, min_len, len);
196             Py_DECREF(arg);
197             return NULL;
198         }
199     }
200 
201     res = (PyStructSequence*) PyStructSequence_New(type);
202     if (res == NULL) {
203         Py_DECREF(arg);
204         return NULL;
205     }
206     for (i = 0; i < len; ++i) {
207         PyObject *v = PySequence_Fast_GET_ITEM(arg, i);
208         Py_INCREF(v);
209         res->ob_item[i] = v;
210     }
211     Py_DECREF(arg);
212     for (; i < max_len; ++i) {
213         if (dict == NULL) {
214             ob = Py_None;
215         }
216         else {
217             ob = _PyDict_GetItemStringWithError(dict,
218                 type->tp_members[i-n_unnamed_fields].name);
219             if (ob == NULL) {
220                 if (PyErr_Occurred()) {
221                     Py_DECREF(res);
222                     return NULL;
223                 }
224                 ob = Py_None;
225             }
226         }
227         Py_INCREF(ob);
228         res->ob_item[i] = ob;
229     }
230 
231     _PyObject_GC_TRACK(res);
232     return (PyObject*) res;
233 }
234 
235 
236 static PyObject *
structseq_repr(PyStructSequence * obj)237 structseq_repr(PyStructSequence *obj)
238 {
239     PyTypeObject *typ = Py_TYPE(obj);
240     _PyUnicodeWriter writer;
241 
242     /* Write "typename(" */
243     PyObject *type_name = PyUnicode_DecodeUTF8(typ->tp_name,
244                                                strlen(typ->tp_name),
245                                                NULL);
246     if (type_name == NULL) {
247         return NULL;
248     }
249 
250     _PyUnicodeWriter_Init(&writer);
251     writer.overallocate = 1;
252     /* count 5 characters per item: "x=1, " */
253     writer.min_length = (PyUnicode_GET_LENGTH(type_name) + 1
254                          + VISIBLE_SIZE(obj) * 5 + 1);
255 
256     if (_PyUnicodeWriter_WriteStr(&writer, type_name) < 0) {
257         Py_DECREF(type_name);
258         goto error;
259     }
260     Py_DECREF(type_name);
261 
262     if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) {
263         goto error;
264     }
265 
266     for (Py_ssize_t i=0; i < VISIBLE_SIZE(obj); i++) {
267         if (i > 0) {
268             /* Write ", " */
269             if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) {
270                 goto error;
271             }
272         }
273 
274         /* Write "name=repr" */
275         const char *name_utf8 = typ->tp_members[i].name;
276         if (name_utf8 == NULL) {
277             PyErr_Format(PyExc_SystemError, "In structseq_repr(), member %zd name is NULL"
278                          " for type %.500s", i, typ->tp_name);
279             goto error;
280         }
281 
282         PyObject *name = PyUnicode_DecodeUTF8(name_utf8, strlen(name_utf8), NULL);
283         if (name == NULL) {
284             goto error;
285         }
286         if (_PyUnicodeWriter_WriteStr(&writer, name) < 0) {
287             Py_DECREF(name);
288             goto error;
289         }
290         Py_DECREF(name);
291 
292         if (_PyUnicodeWriter_WriteChar(&writer, '=') < 0) {
293             goto error;
294         }
295 
296         PyObject *value = PyStructSequence_GET_ITEM(obj, i);
297         assert(value != NULL);
298         PyObject *repr = PyObject_Repr(value);
299         if (repr == NULL) {
300             goto error;
301         }
302         if (_PyUnicodeWriter_WriteStr(&writer, repr) < 0) {
303             Py_DECREF(repr);
304             goto error;
305         }
306         Py_DECREF(repr);
307     }
308 
309     if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) {
310         goto error;
311     }
312 
313     return _PyUnicodeWriter_Finish(&writer);
314 
315 error:
316     _PyUnicodeWriter_Dealloc(&writer);
317     return NULL;
318 }
319 
320 
321 static PyObject *
structseq_reduce(PyStructSequence * self,PyObject * Py_UNUSED (ignored))322 structseq_reduce(PyStructSequence* self, PyObject *Py_UNUSED(ignored))
323 {
324     PyObject* tup = NULL;
325     PyObject* dict = NULL;
326     PyObject* result;
327     Py_ssize_t n_fields, n_visible_fields, n_unnamed_fields, i;
328 
329     n_fields = REAL_SIZE(self);
330     if (n_fields < 0) {
331         return NULL;
332     }
333     n_visible_fields = VISIBLE_SIZE(self);
334     n_unnamed_fields = UNNAMED_FIELDS(self);
335     if (n_unnamed_fields < 0) {
336         return NULL;
337     }
338     tup = _PyTuple_FromArray(self->ob_item, n_visible_fields);
339     if (!tup)
340         goto error;
341 
342     dict = PyDict_New();
343     if (!dict)
344         goto error;
345 
346     for (i = n_visible_fields; i < n_fields; i++) {
347         const char *n = Py_TYPE(self)->tp_members[i-n_unnamed_fields].name;
348         if (PyDict_SetItemString(dict, n, self->ob_item[i]) < 0)
349             goto error;
350     }
351 
352     result = Py_BuildValue("(O(OO))", Py_TYPE(self), tup, dict);
353 
354     Py_DECREF(tup);
355     Py_DECREF(dict);
356 
357     return result;
358 
359 error:
360     Py_XDECREF(tup);
361     Py_XDECREF(dict);
362     return NULL;
363 }
364 
365 static PyMethodDef structseq_methods[] = {
366     {"__reduce__", (PyCFunction)structseq_reduce, METH_NOARGS, NULL},
367     {NULL, NULL}
368 };
369 
370 static Py_ssize_t
count_members(PyStructSequence_Desc * desc,Py_ssize_t * n_unnamed_members)371 count_members(PyStructSequence_Desc *desc, Py_ssize_t *n_unnamed_members) {
372     Py_ssize_t i;
373 
374     *n_unnamed_members = 0;
375     for (i = 0; desc->fields[i].name != NULL; ++i) {
376         if (desc->fields[i].name == PyStructSequence_UnnamedField) {
377             (*n_unnamed_members)++;
378         }
379     }
380     return i;
381 }
382 
383 static int
initialize_structseq_dict(PyStructSequence_Desc * desc,PyObject * dict,Py_ssize_t n_members,Py_ssize_t n_unnamed_members)384 initialize_structseq_dict(PyStructSequence_Desc *desc, PyObject* dict,
385                           Py_ssize_t n_members, Py_ssize_t n_unnamed_members) {
386     PyObject *v;
387 
388 #define SET_DICT_FROM_SIZE(key, value)                                         \
389     do {                                                                       \
390         v = PyLong_FromSsize_t(value);                                         \
391         if (v == NULL) {                                                       \
392             return -1;                                                         \
393         }                                                                      \
394         if (PyDict_SetItemString(dict, key, v) < 0) {                          \
395             Py_DECREF(v);                                                      \
396             return -1;                                                         \
397         }                                                                      \
398         Py_DECREF(v);                                                          \
399     } while (0)
400 
401     SET_DICT_FROM_SIZE(visible_length_key, desc->n_in_sequence);
402     SET_DICT_FROM_SIZE(real_length_key, n_members);
403     SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members);
404 
405     // Prepare and set __match_args__
406     Py_ssize_t i, k;
407     PyObject* keys = PyTuple_New(desc->n_in_sequence);
408     if (keys == NULL) {
409         return -1;
410     }
411 
412     for (i = k = 0; i < desc->n_in_sequence; ++i) {
413         if (desc->fields[i].name == PyStructSequence_UnnamedField) {
414             continue;
415         }
416         PyObject* new_member = PyUnicode_FromString(desc->fields[i].name);
417         if (new_member == NULL) {
418             goto error;
419         }
420         PyTuple_SET_ITEM(keys, k, new_member);
421         k++;
422     }
423 
424     if (_PyTuple_Resize(&keys, k) == -1) {
425         goto error;
426     }
427 
428     if (PyDict_SetItemString(dict, match_args_key, keys) < 0) {
429         goto error;
430     }
431 
432     Py_DECREF(keys);
433     return 0;
434 
435 error:
436     Py_DECREF(keys);
437     return -1;
438 }
439 
440 static void
initialize_members(PyStructSequence_Desc * desc,PyMemberDef * members,Py_ssize_t n_members)441 initialize_members(PyStructSequence_Desc *desc, PyMemberDef* members,
442                    Py_ssize_t n_members) {
443     Py_ssize_t i, k;
444 
445     for (i = k = 0; i < n_members; ++i) {
446         if (desc->fields[i].name == PyStructSequence_UnnamedField) {
447             continue;
448         }
449 
450         /* The names and docstrings in these MemberDefs are statically */
451         /* allocated so it is expected that they'll outlive the MemberDef */
452         members[k].name = desc->fields[i].name;
453         members[k].type = T_OBJECT;
454         members[k].offset = offsetof(PyStructSequence, ob_item)
455           + i * sizeof(PyObject*);
456         members[k].flags = READONLY;
457         members[k].doc = desc->fields[i].doc;
458         k++;
459     }
460     members[k].name = NULL;
461 }
462 
463 
464 int
_PyStructSequence_InitType(PyTypeObject * type,PyStructSequence_Desc * desc,unsigned long tp_flags)465 _PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc,
466                            unsigned long tp_flags)
467 {
468     PyMemberDef *members;
469     Py_ssize_t n_members, n_unnamed_members;
470 
471 #ifdef Py_TRACE_REFS
472     /* if the type object was chained, unchain it first
473        before overwriting its storage */
474     if (type->ob_base.ob_base._ob_next) {
475         _Py_ForgetReference((PyObject *)type);
476     }
477 #endif
478 
479     /* PyTypeObject has already been initialized */
480     if (Py_REFCNT(type) != 0) {
481         PyErr_BadInternalCall();
482         return -1;
483     }
484 
485     type->tp_name = desc->name;
486     type->tp_basicsize = sizeof(PyStructSequence) - sizeof(PyObject *);
487     type->tp_itemsize = sizeof(PyObject *);
488     type->tp_dealloc = (destructor)structseq_dealloc;
489     type->tp_repr = (reprfunc)structseq_repr;
490     type->tp_doc = desc->doc;
491     type->tp_base = &PyTuple_Type;
492     type->tp_methods = structseq_methods;
493     type->tp_new = structseq_new;
494     type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | tp_flags;
495     type->tp_traverse = (traverseproc) structseq_traverse;
496 
497     n_members = count_members(desc, &n_unnamed_members);
498     members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1);
499     if (members == NULL) {
500         PyErr_NoMemory();
501         return -1;
502     }
503     initialize_members(desc, members, n_members);
504     type->tp_members = members;
505 
506     if (PyType_Ready(type) < 0) {
507         PyMem_Free(members);
508         return -1;
509     }
510     Py_INCREF(type);
511 
512     if (initialize_structseq_dict(
513             desc, type->tp_dict, n_members, n_unnamed_members) < 0) {
514         PyMem_Free(members);
515         Py_DECREF(type);
516         return -1;
517     }
518 
519     return 0;
520 }
521 
522 int
PyStructSequence_InitType2(PyTypeObject * type,PyStructSequence_Desc * desc)523 PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc)
524 {
525     return _PyStructSequence_InitType(type, desc, 0);
526 }
527 
528 void
PyStructSequence_InitType(PyTypeObject * type,PyStructSequence_Desc * desc)529 PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc)
530 {
531     (void)PyStructSequence_InitType2(type, desc);
532 }
533 
534 PyTypeObject *
PyStructSequence_NewType(PyStructSequence_Desc * desc)535 PyStructSequence_NewType(PyStructSequence_Desc *desc)
536 {
537     PyMemberDef *members;
538     PyTypeObject *type;
539     PyType_Slot slots[8];
540     PyType_Spec spec;
541     Py_ssize_t n_members, n_unnamed_members;
542 
543     /* Initialize MemberDefs */
544     n_members = count_members(desc, &n_unnamed_members);
545     members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1);
546     if (members == NULL) {
547         PyErr_NoMemory();
548         return NULL;
549     }
550     initialize_members(desc, members, n_members);
551 
552     /* Initialize Slots */
553     slots[0] = (PyType_Slot){Py_tp_dealloc, (destructor)structseq_dealloc};
554     slots[1] = (PyType_Slot){Py_tp_repr, (reprfunc)structseq_repr};
555     slots[2] = (PyType_Slot){Py_tp_doc, (void *)desc->doc};
556     slots[3] = (PyType_Slot){Py_tp_methods, structseq_methods};
557     slots[4] = (PyType_Slot){Py_tp_new, structseq_new};
558     slots[5] = (PyType_Slot){Py_tp_members, members};
559     slots[6] = (PyType_Slot){Py_tp_traverse, (traverseproc)structseq_traverse};
560     slots[7] = (PyType_Slot){0, 0};
561 
562     /* Initialize Spec */
563     /* The name in this PyType_Spec is statically allocated so it is */
564     /* expected that it'll outlive the PyType_Spec */
565     spec.name = desc->name;
566     spec.basicsize = sizeof(PyStructSequence) - sizeof(PyObject *);
567     spec.itemsize = sizeof(PyObject *);
568     spec.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC;
569     spec.slots = slots;
570 
571     type = (PyTypeObject *)PyType_FromSpecWithBases(&spec, (PyObject *)&PyTuple_Type);
572     PyMem_Free(members);
573     if (type == NULL) {
574         return NULL;
575     }
576 
577     if (initialize_structseq_dict(
578             desc, type->tp_dict, n_members, n_unnamed_members) < 0) {
579         Py_DECREF(type);
580         return NULL;
581     }
582 
583     return type;
584 }
585 
_PyStructSequence_Init(void)586 int _PyStructSequence_Init(void)
587 {
588     if (_PyUnicode_FromId(&PyId_n_sequence_fields) == NULL
589         || _PyUnicode_FromId(&PyId_n_fields) == NULL
590         || _PyUnicode_FromId(&PyId_n_unnamed_fields) == NULL)
591     {
592         return -1;
593     }
594     return 0;
595 }
596