• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #include "Python.h"
2 #include "pycore_initconfig.h"
3 #include "pycore_getopt.h"
4 #include "pycore_pystate.h"   /* _PyRuntime_Initialize() */
5 #include <locale.h>       /* setlocale() */
6 
7 
8 #define DECODE_LOCALE_ERR(NAME, LEN) \
9     (((LEN) == -2) \
10      ? _PyStatus_ERR("cannot decode " NAME) \
11      : _PyStatus_NO_MEMORY())
12 
13 
14 /* Forward declarations */
15 static void
16 preconfig_copy(PyPreConfig *config, const PyPreConfig *config2);
17 
18 
19 /* --- File system encoding/errors -------------------------------- */
20 
21 /* The filesystem encoding is chosen by config_init_fs_encoding(),
22    see also initfsencoding().
23 
24    Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors
25    are encoded to UTF-8. */
26 const char *Py_FileSystemDefaultEncoding = NULL;
27 int Py_HasFileSystemDefaultEncoding = 0;
28 const char *Py_FileSystemDefaultEncodeErrors = NULL;
29 int _Py_HasFileSystemDefaultEncodeErrors = 0;
30 
31 void
_Py_ClearFileSystemEncoding(void)32 _Py_ClearFileSystemEncoding(void)
33 {
34     if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) {
35         PyMem_RawFree((char*)Py_FileSystemDefaultEncoding);
36         Py_FileSystemDefaultEncoding = NULL;
37     }
38     if (!_Py_HasFileSystemDefaultEncodeErrors && Py_FileSystemDefaultEncodeErrors) {
39         PyMem_RawFree((char*)Py_FileSystemDefaultEncodeErrors);
40         Py_FileSystemDefaultEncodeErrors = NULL;
41     }
42 }
43 
44 
45 /* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors
46    global configuration variables. */
47 int
_Py_SetFileSystemEncoding(const char * encoding,const char * errors)48 _Py_SetFileSystemEncoding(const char *encoding, const char *errors)
49 {
50     char *encoding2 = _PyMem_RawStrdup(encoding);
51     if (encoding2 == NULL) {
52         return -1;
53     }
54 
55     char *errors2 = _PyMem_RawStrdup(errors);
56     if (errors2 == NULL) {
57         PyMem_RawFree(encoding2);
58         return -1;
59     }
60 
61     _Py_ClearFileSystemEncoding();
62 
63     Py_FileSystemDefaultEncoding = encoding2;
64     Py_HasFileSystemDefaultEncoding = 0;
65 
66     Py_FileSystemDefaultEncodeErrors = errors2;
67     _Py_HasFileSystemDefaultEncodeErrors = 0;
68     return 0;
69 }
70 
71 
72 /* --- _PyArgv ---------------------------------------------------- */
73 
74 /* Decode bytes_argv using Py_DecodeLocale() */
75 PyStatus
_PyArgv_AsWstrList(const _PyArgv * args,PyWideStringList * list)76 _PyArgv_AsWstrList(const _PyArgv *args, PyWideStringList *list)
77 {
78     PyWideStringList wargv = _PyWideStringList_INIT;
79     if (args->use_bytes_argv) {
80         size_t size = sizeof(wchar_t*) * args->argc;
81         wargv.items = (wchar_t **)PyMem_RawMalloc(size);
82         if (wargv.items == NULL) {
83             return _PyStatus_NO_MEMORY();
84         }
85 
86         for (Py_ssize_t i = 0; i < args->argc; i++) {
87             size_t len;
88             wchar_t *arg = Py_DecodeLocale(args->bytes_argv[i], &len);
89             if (arg == NULL) {
90                 _PyWideStringList_Clear(&wargv);
91                 return DECODE_LOCALE_ERR("command line arguments",
92                                          (Py_ssize_t)len);
93             }
94             wargv.items[i] = arg;
95             wargv.length++;
96         }
97 
98         _PyWideStringList_Clear(list);
99         *list = wargv;
100     }
101     else {
102         wargv.length = args->argc;
103         wargv.items = (wchar_t **)args->wchar_argv;
104         if (_PyWideStringList_Copy(list, &wargv) < 0) {
105             return _PyStatus_NO_MEMORY();
106         }
107     }
108     return _PyStatus_OK();
109 }
110 
111 
112 /* --- _PyPreCmdline ------------------------------------------------- */
113 
114 void
_PyPreCmdline_Clear(_PyPreCmdline * cmdline)115 _PyPreCmdline_Clear(_PyPreCmdline *cmdline)
116 {
117     _PyWideStringList_Clear(&cmdline->argv);
118     _PyWideStringList_Clear(&cmdline->xoptions);
119 }
120 
121 
122 PyStatus
_PyPreCmdline_SetArgv(_PyPreCmdline * cmdline,const _PyArgv * args)123 _PyPreCmdline_SetArgv(_PyPreCmdline *cmdline, const _PyArgv *args)
124 {
125     return _PyArgv_AsWstrList(args, &cmdline->argv);
126 }
127 
128 
129 static void
precmdline_get_preconfig(_PyPreCmdline * cmdline,const PyPreConfig * config)130 precmdline_get_preconfig(_PyPreCmdline *cmdline, const PyPreConfig *config)
131 {
132 #define COPY_ATTR(ATTR) \
133     if (config->ATTR != -1) { \
134         cmdline->ATTR = config->ATTR; \
135     }
136 
137     COPY_ATTR(isolated);
138     COPY_ATTR(use_environment);
139     COPY_ATTR(dev_mode);
140 
141 #undef COPY_ATTR
142 }
143 
144 
145 static void
precmdline_set_preconfig(const _PyPreCmdline * cmdline,PyPreConfig * config)146 precmdline_set_preconfig(const _PyPreCmdline *cmdline, PyPreConfig *config)
147 {
148 #define COPY_ATTR(ATTR) \
149     config->ATTR = cmdline->ATTR
150 
151     COPY_ATTR(isolated);
152     COPY_ATTR(use_environment);
153     COPY_ATTR(dev_mode);
154 
155 #undef COPY_ATTR
156 }
157 
158 
159 PyStatus
_PyPreCmdline_SetConfig(const _PyPreCmdline * cmdline,PyConfig * config)160 _PyPreCmdline_SetConfig(const _PyPreCmdline *cmdline, PyConfig *config)
161 {
162 #define COPY_ATTR(ATTR) \
163     config->ATTR = cmdline->ATTR
164 
165     PyStatus status = _PyWideStringList_Extend(&config->xoptions, &cmdline->xoptions);
166     if (_PyStatus_EXCEPTION(status)) {
167         return status;
168     }
169 
170     COPY_ATTR(isolated);
171     COPY_ATTR(use_environment);
172     COPY_ATTR(dev_mode);
173     return _PyStatus_OK();
174 
175 #undef COPY_ATTR
176 }
177 
178 
179 /* Parse the command line arguments */
180 static PyStatus
precmdline_parse_cmdline(_PyPreCmdline * cmdline)181 precmdline_parse_cmdline(_PyPreCmdline *cmdline)
182 {
183     const PyWideStringList *argv = &cmdline->argv;
184 
185     _PyOS_ResetGetOpt();
186     /* Don't log parsing errors into stderr here: PyConfig_Read()
187        is responsible for that */
188     _PyOS_opterr = 0;
189     do {
190         int longindex = -1;
191         int c = _PyOS_GetOpt(argv->length, argv->items, &longindex);
192 
193         if (c == EOF || c == 'c' || c == 'm') {
194             break;
195         }
196 
197         switch (c) {
198         case 'E':
199             cmdline->use_environment = 0;
200             break;
201 
202         case 'I':
203             cmdline->isolated = 1;
204             break;
205 
206         case 'X':
207         {
208             PyStatus status = PyWideStringList_Append(&cmdline->xoptions,
209                                                       _PyOS_optarg);
210             if (_PyStatus_EXCEPTION(status)) {
211                 return status;
212             }
213             break;
214         }
215 
216         default:
217             /* ignore other argument:
218                handled by PyConfig_Read() */
219             break;
220         }
221     } while (1);
222 
223     return _PyStatus_OK();
224 }
225 
226 
227 PyStatus
_PyPreCmdline_Read(_PyPreCmdline * cmdline,const PyPreConfig * preconfig)228 _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig)
229 {
230     precmdline_get_preconfig(cmdline, preconfig);
231 
232     if (preconfig->parse_argv) {
233         PyStatus status = precmdline_parse_cmdline(cmdline);
234         if (_PyStatus_EXCEPTION(status)) {
235             return status;
236         }
237     }
238 
239     /* isolated, use_environment */
240     if (cmdline->isolated < 0) {
241         cmdline->isolated = 0;
242     }
243     if (cmdline->isolated > 0) {
244         cmdline->use_environment = 0;
245     }
246     if (cmdline->use_environment < 0) {
247         cmdline->use_environment = 0;
248     }
249 
250     /* dev_mode */
251     if ((cmdline->dev_mode < 0)
252         && (_Py_get_xoption(&cmdline->xoptions, L"dev")
253             || _Py_GetEnv(cmdline->use_environment, "PYTHONDEVMODE")))
254     {
255         cmdline->dev_mode = 1;
256     }
257     if (cmdline->dev_mode < 0) {
258         cmdline->dev_mode = 0;
259     }
260 
261     assert(cmdline->use_environment >= 0);
262     assert(cmdline->isolated >= 0);
263     assert(cmdline->dev_mode >= 0);
264 
265     return _PyStatus_OK();
266 }
267 
268 
269 /* --- PyPreConfig ----------------------------------------------- */
270 
271 
272 void
_PyPreConfig_InitCompatConfig(PyPreConfig * config)273 _PyPreConfig_InitCompatConfig(PyPreConfig *config)
274 {
275     memset(config, 0, sizeof(*config));
276 
277     config->_config_init = (int)_PyConfig_INIT_COMPAT;
278     config->parse_argv = 0;
279     config->isolated = -1;
280     config->use_environment = -1;
281     config->configure_locale = 1;
282 
283     /* bpo-36443: C locale coercion (PEP 538) and UTF-8 Mode (PEP 540)
284        are disabled by default using the Compat configuration.
285 
286        Py_UTF8Mode=1 enables the UTF-8 mode. PYTHONUTF8 environment variable
287        is ignored (even if use_environment=1). */
288     config->utf8_mode = 0;
289     config->coerce_c_locale = 0;
290     config->coerce_c_locale_warn = 0;
291 
292     config->dev_mode = -1;
293     config->allocator = PYMEM_ALLOCATOR_NOT_SET;
294 #ifdef MS_WINDOWS
295     config->legacy_windows_fs_encoding = -1;
296 #endif
297 }
298 
299 
300 void
PyPreConfig_InitPythonConfig(PyPreConfig * config)301 PyPreConfig_InitPythonConfig(PyPreConfig *config)
302 {
303     _PyPreConfig_InitCompatConfig(config);
304 
305     config->_config_init = (int)_PyConfig_INIT_PYTHON;
306     config->isolated = 0;
307     config->parse_argv = 1;
308     config->use_environment = 1;
309     /* Set to -1 to enable C locale coercion (PEP 538) and UTF-8 Mode (PEP 540)
310        depending on the LC_CTYPE locale, PYTHONUTF8 and PYTHONCOERCECLOCALE
311        environment variables. */
312     config->coerce_c_locale = -1;
313     config->coerce_c_locale_warn = -1;
314     config->utf8_mode = -1;
315 #ifdef MS_WINDOWS
316     config->legacy_windows_fs_encoding = 0;
317 #endif
318 }
319 
320 
321 void
PyPreConfig_InitIsolatedConfig(PyPreConfig * config)322 PyPreConfig_InitIsolatedConfig(PyPreConfig *config)
323 {
324     _PyPreConfig_InitCompatConfig(config);
325 
326     config->_config_init = (int)_PyConfig_INIT_ISOLATED;
327     config->configure_locale = 0;
328     config->isolated = 1;
329     config->use_environment = 0;
330     config->utf8_mode = 0;
331     config->dev_mode = 0;
332 #ifdef MS_WINDOWS
333     config->legacy_windows_fs_encoding = 0;
334 #endif
335 }
336 
337 
338 PyStatus
_PyPreConfig_InitFromPreConfig(PyPreConfig * config,const PyPreConfig * config2)339 _PyPreConfig_InitFromPreConfig(PyPreConfig *config,
340                                const PyPreConfig *config2)
341 {
342     PyPreConfig_InitPythonConfig(config);
343     preconfig_copy(config, config2);
344     return _PyStatus_OK();
345 }
346 
347 
348 void
_PyPreConfig_InitFromConfig(PyPreConfig * preconfig,const PyConfig * config)349 _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config)
350 {
351     _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init;
352     switch (config_init) {
353     case _PyConfig_INIT_PYTHON:
354         PyPreConfig_InitPythonConfig(preconfig);
355         break;
356     case _PyConfig_INIT_ISOLATED:
357         PyPreConfig_InitIsolatedConfig(preconfig);
358         break;
359     case _PyConfig_INIT_COMPAT:
360     default:
361         _PyPreConfig_InitCompatConfig(preconfig);
362     }
363 
364     _PyPreConfig_GetConfig(preconfig, config);
365 }
366 
367 
368 static void
preconfig_copy(PyPreConfig * config,const PyPreConfig * config2)369 preconfig_copy(PyPreConfig *config, const PyPreConfig *config2)
370 {
371 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
372 
373     COPY_ATTR(_config_init);
374     COPY_ATTR(parse_argv);
375     COPY_ATTR(isolated);
376     COPY_ATTR(use_environment);
377     COPY_ATTR(configure_locale);
378     COPY_ATTR(dev_mode);
379     COPY_ATTR(coerce_c_locale);
380     COPY_ATTR(coerce_c_locale_warn);
381     COPY_ATTR(utf8_mode);
382     COPY_ATTR(allocator);
383 #ifdef MS_WINDOWS
384     COPY_ATTR(legacy_windows_fs_encoding);
385 #endif
386 
387 #undef COPY_ATTR
388 }
389 
390 
391 PyObject*
_PyPreConfig_AsDict(const PyPreConfig * config)392 _PyPreConfig_AsDict(const PyPreConfig *config)
393 {
394     PyObject *dict;
395 
396     dict = PyDict_New();
397     if (dict == NULL) {
398         return NULL;
399     }
400 
401 #define SET_ITEM_INT(ATTR) \
402         do { \
403             PyObject *obj = PyLong_FromLong(config->ATTR); \
404             if (obj == NULL) { \
405                 goto fail; \
406             } \
407             int res = PyDict_SetItemString(dict, #ATTR, obj); \
408             Py_DECREF(obj); \
409             if (res < 0) { \
410                 goto fail; \
411             } \
412         } while (0)
413 
414     SET_ITEM_INT(_config_init);
415     SET_ITEM_INT(parse_argv);
416     SET_ITEM_INT(isolated);
417     SET_ITEM_INT(use_environment);
418     SET_ITEM_INT(configure_locale);
419     SET_ITEM_INT(coerce_c_locale);
420     SET_ITEM_INT(coerce_c_locale_warn);
421     SET_ITEM_INT(utf8_mode);
422 #ifdef MS_WINDOWS
423     SET_ITEM_INT(legacy_windows_fs_encoding);
424 #endif
425     SET_ITEM_INT(dev_mode);
426     SET_ITEM_INT(allocator);
427     return dict;
428 
429 fail:
430     Py_DECREF(dict);
431     return NULL;
432 
433 #undef SET_ITEM_INT
434 }
435 
436 
437 void
_PyPreConfig_GetConfig(PyPreConfig * preconfig,const PyConfig * config)438 _PyPreConfig_GetConfig(PyPreConfig *preconfig, const PyConfig *config)
439 {
440 #define COPY_ATTR(ATTR) \
441     if (config->ATTR != -1) { \
442         preconfig->ATTR = config->ATTR; \
443     }
444 
445     COPY_ATTR(parse_argv);
446     COPY_ATTR(isolated);
447     COPY_ATTR(use_environment);
448     COPY_ATTR(dev_mode);
449 
450 #undef COPY_ATTR
451 }
452 
453 
454 static void
preconfig_get_global_vars(PyPreConfig * config)455 preconfig_get_global_vars(PyPreConfig *config)
456 {
457     if (config->_config_init != _PyConfig_INIT_COMPAT) {
458         /* Python and Isolated configuration ignore global variables */
459         return;
460     }
461 
462 #define COPY_FLAG(ATTR, VALUE) \
463     if (config->ATTR < 0) { \
464         config->ATTR = VALUE; \
465     }
466 #define COPY_NOT_FLAG(ATTR, VALUE) \
467     if (config->ATTR < 0) { \
468         config->ATTR = !(VALUE); \
469     }
470 
471     COPY_FLAG(isolated, Py_IsolatedFlag);
472     COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag);
473     if (Py_UTF8Mode > 0) {
474         config->utf8_mode = Py_UTF8Mode;
475     }
476 #ifdef MS_WINDOWS
477     COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag);
478 #endif
479 
480 #undef COPY_FLAG
481 #undef COPY_NOT_FLAG
482 }
483 
484 
485 static void
preconfig_set_global_vars(const PyPreConfig * config)486 preconfig_set_global_vars(const PyPreConfig *config)
487 {
488 #define COPY_FLAG(ATTR, VAR) \
489     if (config->ATTR >= 0) { \
490         VAR = config->ATTR; \
491     }
492 #define COPY_NOT_FLAG(ATTR, VAR) \
493     if (config->ATTR >= 0) { \
494         VAR = !config->ATTR; \
495     }
496 
497     COPY_FLAG(isolated, Py_IsolatedFlag);
498     COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag);
499 #ifdef MS_WINDOWS
500     COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag);
501 #endif
502     COPY_FLAG(utf8_mode, Py_UTF8Mode);
503 
504 #undef COPY_FLAG
505 #undef COPY_NOT_FLAG
506 }
507 
508 
509 const char*
_Py_GetEnv(int use_environment,const char * name)510 _Py_GetEnv(int use_environment, const char *name)
511 {
512     assert(use_environment >= 0);
513 
514     if (!use_environment) {
515         return NULL;
516     }
517 
518     const char *var = getenv(name);
519     if (var && var[0] != '\0') {
520         return var;
521     }
522     else {
523         return NULL;
524     }
525 }
526 
527 
528 int
_Py_str_to_int(const char * str,int * result)529 _Py_str_to_int(const char *str, int *result)
530 {
531     const char *endptr = str;
532     errno = 0;
533     long value = strtol(str, (char **)&endptr, 10);
534     if (*endptr != '\0' || errno == ERANGE) {
535         return -1;
536     }
537     if (value < INT_MIN || value > INT_MAX) {
538         return -1;
539     }
540 
541     *result = (int)value;
542     return 0;
543 }
544 
545 
546 void
_Py_get_env_flag(int use_environment,int * flag,const char * name)547 _Py_get_env_flag(int use_environment, int *flag, const char *name)
548 {
549     const char *var = _Py_GetEnv(use_environment, name);
550     if (!var) {
551         return;
552     }
553     int value;
554     if (_Py_str_to_int(var, &value) < 0 || value < 0) {
555         /* PYTHONDEBUG=text and PYTHONDEBUG=-2 behave as PYTHONDEBUG=1 */
556         value = 1;
557     }
558     if (*flag < value) {
559         *flag = value;
560     }
561 }
562 
563 
564 const wchar_t*
_Py_get_xoption(const PyWideStringList * xoptions,const wchar_t * name)565 _Py_get_xoption(const PyWideStringList *xoptions, const wchar_t *name)
566 {
567     for (Py_ssize_t i=0; i < xoptions->length; i++) {
568         const wchar_t *option = xoptions->items[i];
569         size_t len;
570         wchar_t *sep = wcschr(option, L'=');
571         if (sep != NULL) {
572             len = (sep - option);
573         }
574         else {
575             len = wcslen(option);
576         }
577         if (wcsncmp(option, name, len) == 0 && name[len] == L'\0') {
578             return option;
579         }
580     }
581     return NULL;
582 }
583 
584 
585 static PyStatus
preconfig_init_utf8_mode(PyPreConfig * config,const _PyPreCmdline * cmdline)586 preconfig_init_utf8_mode(PyPreConfig *config, const _PyPreCmdline *cmdline)
587 {
588 #ifdef MS_WINDOWS
589     if (config->legacy_windows_fs_encoding) {
590         config->utf8_mode = 0;
591     }
592 #endif
593 
594     if (config->utf8_mode >= 0) {
595         return _PyStatus_OK();
596     }
597 
598     const wchar_t *xopt;
599     xopt = _Py_get_xoption(&cmdline->xoptions, L"utf8");
600     if (xopt) {
601         wchar_t *sep = wcschr(xopt, L'=');
602         if (sep) {
603             xopt = sep + 1;
604             if (wcscmp(xopt, L"1") == 0) {
605                 config->utf8_mode = 1;
606             }
607             else if (wcscmp(xopt, L"0") == 0) {
608                 config->utf8_mode = 0;
609             }
610             else {
611                 return _PyStatus_ERR("invalid -X utf8 option value");
612             }
613         }
614         else {
615             config->utf8_mode = 1;
616         }
617         return _PyStatus_OK();
618     }
619 
620     const char *opt = _Py_GetEnv(config->use_environment, "PYTHONUTF8");
621     if (opt) {
622         if (strcmp(opt, "1") == 0) {
623             config->utf8_mode = 1;
624         }
625         else if (strcmp(opt, "0") == 0) {
626             config->utf8_mode = 0;
627         }
628         else {
629             return _PyStatus_ERR("invalid PYTHONUTF8 environment "
630                                 "variable value");
631         }
632         return _PyStatus_OK();
633     }
634 
635 
636 #ifndef MS_WINDOWS
637     if (config->utf8_mode < 0) {
638         /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */
639         const char *ctype_loc = setlocale(LC_CTYPE, NULL);
640         if (ctype_loc != NULL
641            && (strcmp(ctype_loc, "C") == 0
642                || strcmp(ctype_loc, "POSIX") == 0))
643         {
644             config->utf8_mode = 1;
645         }
646     }
647 #endif
648 
649     if (config->utf8_mode < 0) {
650         config->utf8_mode = 0;
651     }
652     return _PyStatus_OK();
653 }
654 
655 
656 static void
preconfig_init_coerce_c_locale(PyPreConfig * config)657 preconfig_init_coerce_c_locale(PyPreConfig *config)
658 {
659     if (!config->configure_locale) {
660         config->coerce_c_locale = 0;
661         config->coerce_c_locale_warn = 0;
662         return;
663     }
664 
665     const char *env = _Py_GetEnv(config->use_environment, "PYTHONCOERCECLOCALE");
666     if (env) {
667         if (strcmp(env, "0") == 0) {
668             if (config->coerce_c_locale < 0) {
669                 config->coerce_c_locale = 0;
670             }
671         }
672         else if (strcmp(env, "warn") == 0) {
673             if (config->coerce_c_locale_warn < 0) {
674                 config->coerce_c_locale_warn = 1;
675             }
676         }
677         else {
678             if (config->coerce_c_locale < 0) {
679                 config->coerce_c_locale = 1;
680             }
681         }
682     }
683 
684     /* Test if coerce_c_locale equals to -1 or equals to 1:
685        PYTHONCOERCECLOCALE=1 doesn't imply that the C locale is always coerced.
686        It is only coerced if if the LC_CTYPE locale is "C". */
687     if (config->coerce_c_locale < 0 || config->coerce_c_locale == 1) {
688         /* The C locale enables the C locale coercion (PEP 538) */
689         if (_Py_LegacyLocaleDetected(0)) {
690             config->coerce_c_locale = 2;
691         }
692         else {
693             config->coerce_c_locale = 0;
694         }
695     }
696 
697     if (config->coerce_c_locale_warn < 0) {
698         config->coerce_c_locale_warn = 0;
699     }
700 }
701 
702 
703 static PyStatus
preconfig_init_allocator(PyPreConfig * config)704 preconfig_init_allocator(PyPreConfig *config)
705 {
706     if (config->allocator == PYMEM_ALLOCATOR_NOT_SET) {
707         /* bpo-34247. The PYTHONMALLOC environment variable has the priority
708            over PYTHONDEV env var and "-X dev" command line option.
709            For example, PYTHONMALLOC=malloc PYTHONDEVMODE=1 sets the memory
710            allocators to "malloc" (and not to "debug"). */
711         const char *envvar = _Py_GetEnv(config->use_environment, "PYTHONMALLOC");
712         if (envvar) {
713             PyMemAllocatorName name;
714             if (_PyMem_GetAllocatorName(envvar, &name) < 0) {
715                 return _PyStatus_ERR("PYTHONMALLOC: unknown allocator");
716             }
717             config->allocator = (int)name;
718         }
719     }
720 
721     if (config->dev_mode && config->allocator == PYMEM_ALLOCATOR_NOT_SET) {
722         config->allocator = PYMEM_ALLOCATOR_DEBUG;
723     }
724     return _PyStatus_OK();
725 }
726 
727 
728 static PyStatus
preconfig_read(PyPreConfig * config,_PyPreCmdline * cmdline)729 preconfig_read(PyPreConfig *config, _PyPreCmdline *cmdline)
730 {
731     PyStatus status;
732 
733     status = _PyPreCmdline_Read(cmdline, config);
734     if (_PyStatus_EXCEPTION(status)) {
735         return status;
736     }
737 
738     precmdline_set_preconfig(cmdline, config);
739 
740     /* legacy_windows_fs_encoding, coerce_c_locale, utf8_mode */
741 #ifdef MS_WINDOWS
742     _Py_get_env_flag(config->use_environment,
743                      &config->legacy_windows_fs_encoding,
744                      "PYTHONLEGACYWINDOWSFSENCODING");
745 #endif
746 
747     preconfig_init_coerce_c_locale(config);
748 
749     status = preconfig_init_utf8_mode(config, cmdline);
750     if (_PyStatus_EXCEPTION(status)) {
751         return status;
752     }
753 
754     /* allocator */
755     status = preconfig_init_allocator(config);
756     if (_PyStatus_EXCEPTION(status)) {
757         return status;
758     }
759 
760     assert(config->coerce_c_locale >= 0);
761     assert(config->coerce_c_locale_warn >= 0);
762 #ifdef MS_WINDOWS
763     assert(config->legacy_windows_fs_encoding >= 0);
764 #endif
765     assert(config->utf8_mode >= 0);
766     assert(config->isolated >= 0);
767     assert(config->use_environment >= 0);
768     assert(config->dev_mode >= 0);
769 
770     return _PyStatus_OK();
771 }
772 
773 
774 /* Read the configuration from:
775 
776    - command line arguments
777    - environment variables
778    - Py_xxx global configuration variables
779    - the LC_CTYPE locale */
780 PyStatus
_PyPreConfig_Read(PyPreConfig * config,const _PyArgv * args)781 _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args)
782 {
783     PyStatus status;
784 
785     status = _PyRuntime_Initialize();
786     if (_PyStatus_EXCEPTION(status)) {
787         return status;
788     }
789 
790     preconfig_get_global_vars(config);
791 
792     /* Copy LC_CTYPE locale, since it's modified later */
793     const char *loc = setlocale(LC_CTYPE, NULL);
794     if (loc == NULL) {
795         return _PyStatus_ERR("failed to LC_CTYPE locale");
796     }
797     char *init_ctype_locale = _PyMem_RawStrdup(loc);
798     if (init_ctype_locale == NULL) {
799         return _PyStatus_NO_MEMORY();
800     }
801 
802     /* Save the config to be able to restore it if encodings change */
803     PyPreConfig save_config;
804 
805     status = _PyPreConfig_InitFromPreConfig(&save_config, config);
806     if (_PyStatus_EXCEPTION(status)) {
807         return status;
808     }
809 
810     /* Set LC_CTYPE to the user preferred locale */
811     if (config->configure_locale) {
812         _Py_SetLocaleFromEnv(LC_CTYPE);
813     }
814 
815     _PyPreCmdline cmdline = _PyPreCmdline_INIT;
816     int init_utf8_mode = Py_UTF8Mode;
817 #ifdef MS_WINDOWS
818     int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag;
819 #endif
820 
821     if (args) {
822         status = _PyPreCmdline_SetArgv(&cmdline, args);
823         if (_PyStatus_EXCEPTION(status)) {
824             goto done;
825         }
826     }
827 
828     int locale_coerced = 0;
829     int loops = 0;
830 
831     while (1) {
832         int utf8_mode = config->utf8_mode;
833 
834         /* Watchdog to prevent an infinite loop */
835         loops++;
836         if (loops == 3) {
837             status = _PyStatus_ERR("Encoding changed twice while "
838                                "reading the configuration");
839             goto done;
840         }
841 
842         /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend
843            on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */
844         Py_UTF8Mode = config->utf8_mode;
845 #ifdef MS_WINDOWS
846         Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding;
847 #endif
848 
849         status = preconfig_read(config, &cmdline);
850         if (_PyStatus_EXCEPTION(status)) {
851             goto done;
852         }
853 
854         /* The legacy C locale assumes ASCII as the default text encoding, which
855          * causes problems not only for the CPython runtime, but also other
856          * components like GNU readline.
857          *
858          * Accordingly, when the CLI detects it, it attempts to coerce it to a
859          * more capable UTF-8 based alternative.
860          *
861          * See the documentation of the PYTHONCOERCECLOCALE setting for more
862          * details.
863          */
864         int encoding_changed = 0;
865         if (config->coerce_c_locale && !locale_coerced) {
866             locale_coerced = 1;
867             _Py_CoerceLegacyLocale(0);
868             encoding_changed = 1;
869         }
870 
871         if (utf8_mode == -1) {
872             if (config->utf8_mode == 1) {
873                 /* UTF-8 Mode enabled */
874                 encoding_changed = 1;
875             }
876         }
877         else {
878             if (config->utf8_mode != utf8_mode) {
879                 encoding_changed = 1;
880             }
881         }
882 
883         if (!encoding_changed) {
884             break;
885         }
886 
887         /* Reset the configuration before reading again the configuration,
888            just keep UTF-8 Mode value. */
889         int new_utf8_mode = config->utf8_mode;
890         int new_coerce_c_locale = config->coerce_c_locale;
891         preconfig_copy(config, &save_config);
892         config->utf8_mode = new_utf8_mode;
893         config->coerce_c_locale = new_coerce_c_locale;
894 
895         /* The encoding changed: read again the configuration
896            with the new encoding */
897     }
898     status = _PyStatus_OK();
899 
900 done:
901     if (init_ctype_locale != NULL) {
902         setlocale(LC_CTYPE, init_ctype_locale);
903         PyMem_RawFree(init_ctype_locale);
904     }
905     Py_UTF8Mode = init_utf8_mode ;
906 #ifdef MS_WINDOWS
907     Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding;
908 #endif
909     _PyPreCmdline_Clear(&cmdline);
910     return status;
911 }
912 
913 
914 /* Write the pre-configuration:
915 
916    - set the memory allocators
917    - set Py_xxx global configuration variables
918    - set the LC_CTYPE locale (coerce C locale, PEP 538) and set the UTF-8 mode
919      (PEP 540)
920 
921    The applied configuration is written into _PyRuntime.preconfig.
922    If the C locale cannot be coerced, set coerce_c_locale to 0.
923 
924    Do nothing if called after Py_Initialize(): ignore the new
925    pre-configuration. */
926 PyStatus
_PyPreConfig_Write(const PyPreConfig * src_config)927 _PyPreConfig_Write(const PyPreConfig *src_config)
928 {
929     PyPreConfig config;
930 
931     PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config);
932     if (_PyStatus_EXCEPTION(status)) {
933         return status;
934     }
935 
936     if (_PyRuntime.core_initialized) {
937         /* bpo-34008: Calling this functions after Py_Initialize() ignores
938            the new configuration. */
939         return _PyStatus_OK();
940     }
941 
942     PyMemAllocatorName name = (PyMemAllocatorName)config.allocator;
943     if (name != PYMEM_ALLOCATOR_NOT_SET) {
944         if (_PyMem_SetupAllocators(name) < 0) {
945             return _PyStatus_ERR("Unknown PYTHONMALLOC allocator");
946         }
947     }
948 
949     preconfig_set_global_vars(&config);
950 
951     if (config.configure_locale) {
952         if (config.coerce_c_locale) {
953             if (!_Py_CoerceLegacyLocale(config.coerce_c_locale_warn)) {
954                 /* C locale not coerced */
955                 config.coerce_c_locale = 0;
956             }
957         }
958 
959         /* Set LC_CTYPE to the user preferred locale */
960         _Py_SetLocaleFromEnv(LC_CTYPE);
961     }
962 
963     /* Write the new pre-configuration into _PyRuntime */
964     preconfig_copy(&_PyRuntime.preconfig, &config);
965 
966     return _PyStatus_OK();
967 }
968