1 #include "Python.h" 2 #include "pycore_getopt.h" // _PyOS_GetOpt() 3 #include "pycore_initconfig.h" // _PyArgv 4 #include "pycore_pymem.h" // _PyMem_GetAllocatorName() 5 #include "pycore_runtime.h" // _PyRuntime_Initialize() 6 #include <locale.h> // setlocale() 7 8 9 #define DECODE_LOCALE_ERR(NAME, LEN) \ 10 (((LEN) == -2) \ 11 ? _PyStatus_ERR("cannot decode " NAME) \ 12 : _PyStatus_NO_MEMORY()) 13 14 15 /* Forward declarations */ 16 static void 17 preconfig_copy(PyPreConfig *config, const PyPreConfig *config2); 18 19 20 /* --- File system encoding/errors -------------------------------- */ 21 22 const char *Py_FileSystemDefaultEncoding = NULL; 23 int Py_HasFileSystemDefaultEncoding = 0; 24 const char *Py_FileSystemDefaultEncodeErrors = NULL; 25 int _Py_HasFileSystemDefaultEncodeErrors = 0; 26 27 void _Py_ClearFileSystemEncoding(void)28 _Py_ClearFileSystemEncoding(void) 29 { 30 if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { 31 PyMem_RawFree((char*)Py_FileSystemDefaultEncoding); 32 Py_FileSystemDefaultEncoding = NULL; 33 } 34 if (!_Py_HasFileSystemDefaultEncodeErrors && Py_FileSystemDefaultEncodeErrors) { 35 PyMem_RawFree((char*)Py_FileSystemDefaultEncodeErrors); 36 Py_FileSystemDefaultEncodeErrors = NULL; 37 } 38 } 39 40 41 /* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors 42 global configuration variables to PyConfig.filesystem_encoding and 43 PyConfig.filesystem_errors (encoded to UTF-8). 44 45 Function called by _PyUnicode_InitEncodings(). */ 46 int _Py_SetFileSystemEncoding(const char * encoding,const char * errors)47 _Py_SetFileSystemEncoding(const char *encoding, const char *errors) 48 { 49 char *encoding2 = _PyMem_RawStrdup(encoding); 50 if (encoding2 == NULL) { 51 return -1; 52 } 53 54 char *errors2 = _PyMem_RawStrdup(errors); 55 if (errors2 == NULL) { 56 PyMem_RawFree(encoding2); 57 return -1; 58 } 59 60 _Py_ClearFileSystemEncoding(); 61 62 Py_FileSystemDefaultEncoding = encoding2; 63 Py_HasFileSystemDefaultEncoding = 0; 64 65 Py_FileSystemDefaultEncodeErrors = errors2; 66 _Py_HasFileSystemDefaultEncodeErrors = 0; 67 return 0; 68 } 69 70 71 /* --- _PyArgv ---------------------------------------------------- */ 72 73 /* Decode bytes_argv using Py_DecodeLocale() */ 74 PyStatus _PyArgv_AsWstrList(const _PyArgv * args,PyWideStringList * list)75 _PyArgv_AsWstrList(const _PyArgv *args, PyWideStringList *list) 76 { 77 PyWideStringList wargv = _PyWideStringList_INIT; 78 if (args->use_bytes_argv) { 79 size_t size = sizeof(wchar_t*) * args->argc; 80 wargv.items = (wchar_t **)PyMem_RawMalloc(size); 81 if (wargv.items == NULL) { 82 return _PyStatus_NO_MEMORY(); 83 } 84 85 for (Py_ssize_t i = 0; i < args->argc; i++) { 86 size_t len; 87 wchar_t *arg = Py_DecodeLocale(args->bytes_argv[i], &len); 88 if (arg == NULL) { 89 _PyWideStringList_Clear(&wargv); 90 return DECODE_LOCALE_ERR("command line arguments", 91 (Py_ssize_t)len); 92 } 93 wargv.items[i] = arg; 94 wargv.length++; 95 } 96 97 _PyWideStringList_Clear(list); 98 *list = wargv; 99 } 100 else { 101 wargv.length = args->argc; 102 wargv.items = (wchar_t **)args->wchar_argv; 103 if (_PyWideStringList_Copy(list, &wargv) < 0) { 104 return _PyStatus_NO_MEMORY(); 105 } 106 } 107 return _PyStatus_OK(); 108 } 109 110 111 /* --- _PyPreCmdline ------------------------------------------------- */ 112 113 void _PyPreCmdline_Clear(_PyPreCmdline * cmdline)114 _PyPreCmdline_Clear(_PyPreCmdline *cmdline) 115 { 116 _PyWideStringList_Clear(&cmdline->argv); 117 _PyWideStringList_Clear(&cmdline->xoptions); 118 } 119 120 121 PyStatus _PyPreCmdline_SetArgv(_PyPreCmdline * cmdline,const _PyArgv * args)122 _PyPreCmdline_SetArgv(_PyPreCmdline *cmdline, const _PyArgv *args) 123 { 124 return _PyArgv_AsWstrList(args, &cmdline->argv); 125 } 126 127 128 static void precmdline_get_preconfig(_PyPreCmdline * cmdline,const PyPreConfig * config)129 precmdline_get_preconfig(_PyPreCmdline *cmdline, const PyPreConfig *config) 130 { 131 #define COPY_ATTR(ATTR) \ 132 if (config->ATTR != -1) { \ 133 cmdline->ATTR = config->ATTR; \ 134 } 135 136 COPY_ATTR(isolated); 137 COPY_ATTR(use_environment); 138 COPY_ATTR(dev_mode); 139 140 #undef COPY_ATTR 141 } 142 143 144 static void precmdline_set_preconfig(const _PyPreCmdline * cmdline,PyPreConfig * config)145 precmdline_set_preconfig(const _PyPreCmdline *cmdline, PyPreConfig *config) 146 { 147 #define COPY_ATTR(ATTR) \ 148 config->ATTR = cmdline->ATTR 149 150 COPY_ATTR(isolated); 151 COPY_ATTR(use_environment); 152 COPY_ATTR(dev_mode); 153 154 #undef COPY_ATTR 155 } 156 157 158 PyStatus _PyPreCmdline_SetConfig(const _PyPreCmdline * cmdline,PyConfig * config)159 _PyPreCmdline_SetConfig(const _PyPreCmdline *cmdline, PyConfig *config) 160 { 161 #define COPY_ATTR(ATTR) \ 162 config->ATTR = cmdline->ATTR 163 164 PyStatus status = _PyWideStringList_Extend(&config->xoptions, &cmdline->xoptions); 165 if (_PyStatus_EXCEPTION(status)) { 166 return status; 167 } 168 169 COPY_ATTR(isolated); 170 COPY_ATTR(use_environment); 171 COPY_ATTR(dev_mode); 172 COPY_ATTR(warn_default_encoding); 173 return _PyStatus_OK(); 174 175 #undef COPY_ATTR 176 } 177 178 179 /* Parse the command line arguments */ 180 static PyStatus precmdline_parse_cmdline(_PyPreCmdline * cmdline)181 precmdline_parse_cmdline(_PyPreCmdline *cmdline) 182 { 183 const PyWideStringList *argv = &cmdline->argv; 184 185 _PyOS_ResetGetOpt(); 186 /* Don't log parsing errors into stderr here: PyConfig_Read() 187 is responsible for that */ 188 _PyOS_opterr = 0; 189 do { 190 int longindex = -1; 191 int c = _PyOS_GetOpt(argv->length, argv->items, &longindex); 192 193 if (c == EOF || c == 'c' || c == 'm') { 194 break; 195 } 196 197 switch (c) { 198 case 'E': 199 cmdline->use_environment = 0; 200 break; 201 202 case 'I': 203 cmdline->isolated = 1; 204 break; 205 206 case 'X': 207 { 208 PyStatus status = PyWideStringList_Append(&cmdline->xoptions, 209 _PyOS_optarg); 210 if (_PyStatus_EXCEPTION(status)) { 211 return status; 212 } 213 break; 214 } 215 216 default: 217 /* ignore other argument: 218 handled by PyConfig_Read() */ 219 break; 220 } 221 } while (1); 222 223 return _PyStatus_OK(); 224 } 225 226 227 PyStatus _PyPreCmdline_Read(_PyPreCmdline * cmdline,const PyPreConfig * preconfig)228 _PyPreCmdline_Read(_PyPreCmdline *cmdline, const PyPreConfig *preconfig) 229 { 230 precmdline_get_preconfig(cmdline, preconfig); 231 232 if (preconfig->parse_argv) { 233 PyStatus status = precmdline_parse_cmdline(cmdline); 234 if (_PyStatus_EXCEPTION(status)) { 235 return status; 236 } 237 } 238 239 /* isolated, use_environment */ 240 if (cmdline->isolated < 0) { 241 cmdline->isolated = 0; 242 } 243 if (cmdline->isolated > 0) { 244 cmdline->use_environment = 0; 245 } 246 if (cmdline->use_environment < 0) { 247 cmdline->use_environment = 0; 248 } 249 250 /* dev_mode */ 251 if ((cmdline->dev_mode < 0) 252 && (_Py_get_xoption(&cmdline->xoptions, L"dev") 253 || _Py_GetEnv(cmdline->use_environment, "PYTHONDEVMODE"))) 254 { 255 cmdline->dev_mode = 1; 256 } 257 if (cmdline->dev_mode < 0) { 258 cmdline->dev_mode = 0; 259 } 260 261 // warn_default_encoding 262 if (_Py_get_xoption(&cmdline->xoptions, L"warn_default_encoding") 263 || _Py_GetEnv(cmdline->use_environment, "PYTHONWARNDEFAULTENCODING")) 264 { 265 cmdline->warn_default_encoding = 1; 266 } 267 268 assert(cmdline->use_environment >= 0); 269 assert(cmdline->isolated >= 0); 270 assert(cmdline->dev_mode >= 0); 271 assert(cmdline->warn_default_encoding >= 0); 272 273 return _PyStatus_OK(); 274 } 275 276 277 /* --- PyPreConfig ----------------------------------------------- */ 278 279 280 void _PyPreConfig_InitCompatConfig(PyPreConfig * config)281 _PyPreConfig_InitCompatConfig(PyPreConfig *config) 282 { 283 memset(config, 0, sizeof(*config)); 284 285 config->_config_init = (int)_PyConfig_INIT_COMPAT; 286 config->parse_argv = 0; 287 config->isolated = -1; 288 config->use_environment = -1; 289 config->configure_locale = 1; 290 291 /* bpo-36443: C locale coercion (PEP 538) and UTF-8 Mode (PEP 540) 292 are disabled by default using the Compat configuration. 293 294 Py_UTF8Mode=1 enables the UTF-8 mode. PYTHONUTF8 environment variable 295 is ignored (even if use_environment=1). */ 296 config->utf8_mode = 0; 297 config->coerce_c_locale = 0; 298 config->coerce_c_locale_warn = 0; 299 300 config->dev_mode = -1; 301 #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS 302 /* bpo-40512: pymalloc is not compatible with subinterpreters, 303 force usage of libc malloc() which is thread-safe. */ 304 #ifdef Py_DEBUG 305 config->allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; 306 #else 307 config->allocator = PYMEM_ALLOCATOR_MALLOC; 308 #endif 309 #else 310 config->allocator = PYMEM_ALLOCATOR_NOT_SET; 311 #endif 312 #ifdef MS_WINDOWS 313 config->legacy_windows_fs_encoding = -1; 314 #endif 315 } 316 317 318 void PyPreConfig_InitPythonConfig(PyPreConfig * config)319 PyPreConfig_InitPythonConfig(PyPreConfig *config) 320 { 321 _PyPreConfig_InitCompatConfig(config); 322 323 config->_config_init = (int)_PyConfig_INIT_PYTHON; 324 config->isolated = 0; 325 config->parse_argv = 1; 326 config->use_environment = 1; 327 /* Set to -1 to enable C locale coercion (PEP 538) and UTF-8 Mode (PEP 540) 328 depending on the LC_CTYPE locale, PYTHONUTF8 and PYTHONCOERCECLOCALE 329 environment variables. */ 330 config->coerce_c_locale = -1; 331 config->coerce_c_locale_warn = -1; 332 config->utf8_mode = -1; 333 #ifdef MS_WINDOWS 334 config->legacy_windows_fs_encoding = 0; 335 #endif 336 } 337 338 339 void PyPreConfig_InitIsolatedConfig(PyPreConfig * config)340 PyPreConfig_InitIsolatedConfig(PyPreConfig *config) 341 { 342 _PyPreConfig_InitCompatConfig(config); 343 344 config->_config_init = (int)_PyConfig_INIT_ISOLATED; 345 config->configure_locale = 0; 346 config->isolated = 1; 347 config->use_environment = 0; 348 config->utf8_mode = 0; 349 config->dev_mode = 0; 350 #ifdef MS_WINDOWS 351 config->legacy_windows_fs_encoding = 0; 352 #endif 353 } 354 355 356 PyStatus _PyPreConfig_InitFromPreConfig(PyPreConfig * config,const PyPreConfig * config2)357 _PyPreConfig_InitFromPreConfig(PyPreConfig *config, 358 const PyPreConfig *config2) 359 { 360 PyPreConfig_InitPythonConfig(config); 361 preconfig_copy(config, config2); 362 return _PyStatus_OK(); 363 } 364 365 366 void _PyPreConfig_InitFromConfig(PyPreConfig * preconfig,const PyConfig * config)367 _PyPreConfig_InitFromConfig(PyPreConfig *preconfig, const PyConfig *config) 368 { 369 _PyConfigInitEnum config_init = (_PyConfigInitEnum)config->_config_init; 370 switch (config_init) { 371 case _PyConfig_INIT_PYTHON: 372 PyPreConfig_InitPythonConfig(preconfig); 373 break; 374 case _PyConfig_INIT_ISOLATED: 375 PyPreConfig_InitIsolatedConfig(preconfig); 376 break; 377 case _PyConfig_INIT_COMPAT: 378 default: 379 _PyPreConfig_InitCompatConfig(preconfig); 380 } 381 382 _PyPreConfig_GetConfig(preconfig, config); 383 } 384 385 386 static void preconfig_copy(PyPreConfig * config,const PyPreConfig * config2)387 preconfig_copy(PyPreConfig *config, const PyPreConfig *config2) 388 { 389 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR 390 391 COPY_ATTR(_config_init); 392 COPY_ATTR(parse_argv); 393 COPY_ATTR(isolated); 394 COPY_ATTR(use_environment); 395 COPY_ATTR(configure_locale); 396 COPY_ATTR(dev_mode); 397 COPY_ATTR(coerce_c_locale); 398 COPY_ATTR(coerce_c_locale_warn); 399 COPY_ATTR(utf8_mode); 400 COPY_ATTR(allocator); 401 #ifdef MS_WINDOWS 402 COPY_ATTR(legacy_windows_fs_encoding); 403 #endif 404 405 #undef COPY_ATTR 406 } 407 408 409 PyObject* _PyPreConfig_AsDict(const PyPreConfig * config)410 _PyPreConfig_AsDict(const PyPreConfig *config) 411 { 412 PyObject *dict; 413 414 dict = PyDict_New(); 415 if (dict == NULL) { 416 return NULL; 417 } 418 419 #define SET_ITEM_INT(ATTR) \ 420 do { \ 421 PyObject *obj = PyLong_FromLong(config->ATTR); \ 422 if (obj == NULL) { \ 423 goto fail; \ 424 } \ 425 int res = PyDict_SetItemString(dict, #ATTR, obj); \ 426 Py_DECREF(obj); \ 427 if (res < 0) { \ 428 goto fail; \ 429 } \ 430 } while (0) 431 432 SET_ITEM_INT(_config_init); 433 SET_ITEM_INT(parse_argv); 434 SET_ITEM_INT(isolated); 435 SET_ITEM_INT(use_environment); 436 SET_ITEM_INT(configure_locale); 437 SET_ITEM_INT(coerce_c_locale); 438 SET_ITEM_INT(coerce_c_locale_warn); 439 SET_ITEM_INT(utf8_mode); 440 #ifdef MS_WINDOWS 441 SET_ITEM_INT(legacy_windows_fs_encoding); 442 #endif 443 SET_ITEM_INT(dev_mode); 444 SET_ITEM_INT(allocator); 445 return dict; 446 447 fail: 448 Py_DECREF(dict); 449 return NULL; 450 451 #undef SET_ITEM_INT 452 } 453 454 455 void _PyPreConfig_GetConfig(PyPreConfig * preconfig,const PyConfig * config)456 _PyPreConfig_GetConfig(PyPreConfig *preconfig, const PyConfig *config) 457 { 458 #define COPY_ATTR(ATTR) \ 459 if (config->ATTR != -1) { \ 460 preconfig->ATTR = config->ATTR; \ 461 } 462 463 COPY_ATTR(parse_argv); 464 COPY_ATTR(isolated); 465 COPY_ATTR(use_environment); 466 COPY_ATTR(dev_mode); 467 468 #undef COPY_ATTR 469 } 470 471 472 static void preconfig_get_global_vars(PyPreConfig * config)473 preconfig_get_global_vars(PyPreConfig *config) 474 { 475 if (config->_config_init != _PyConfig_INIT_COMPAT) { 476 /* Python and Isolated configuration ignore global variables */ 477 return; 478 } 479 480 #define COPY_FLAG(ATTR, VALUE) \ 481 if (config->ATTR < 0) { \ 482 config->ATTR = VALUE; \ 483 } 484 #define COPY_NOT_FLAG(ATTR, VALUE) \ 485 if (config->ATTR < 0) { \ 486 config->ATTR = !(VALUE); \ 487 } 488 489 COPY_FLAG(isolated, Py_IsolatedFlag); 490 COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); 491 if (Py_UTF8Mode > 0) { 492 config->utf8_mode = Py_UTF8Mode; 493 } 494 #ifdef MS_WINDOWS 495 COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); 496 #endif 497 498 #undef COPY_FLAG 499 #undef COPY_NOT_FLAG 500 } 501 502 503 static void preconfig_set_global_vars(const PyPreConfig * config)504 preconfig_set_global_vars(const PyPreConfig *config) 505 { 506 #define COPY_FLAG(ATTR, VAR) \ 507 if (config->ATTR >= 0) { \ 508 VAR = config->ATTR; \ 509 } 510 #define COPY_NOT_FLAG(ATTR, VAR) \ 511 if (config->ATTR >= 0) { \ 512 VAR = !config->ATTR; \ 513 } 514 515 COPY_FLAG(isolated, Py_IsolatedFlag); 516 COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); 517 #ifdef MS_WINDOWS 518 COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); 519 #endif 520 COPY_FLAG(utf8_mode, Py_UTF8Mode); 521 522 #undef COPY_FLAG 523 #undef COPY_NOT_FLAG 524 } 525 526 527 const char* _Py_GetEnv(int use_environment,const char * name)528 _Py_GetEnv(int use_environment, const char *name) 529 { 530 assert(use_environment >= 0); 531 532 if (!use_environment) { 533 return NULL; 534 } 535 536 const char *var = getenv(name); 537 if (var && var[0] != '\0') { 538 return var; 539 } 540 else { 541 return NULL; 542 } 543 } 544 545 546 int _Py_str_to_int(const char * str,int * result)547 _Py_str_to_int(const char *str, int *result) 548 { 549 const char *endptr = str; 550 errno = 0; 551 long value = strtol(str, (char **)&endptr, 10); 552 if (*endptr != '\0' || errno == ERANGE) { 553 return -1; 554 } 555 if (value < INT_MIN || value > INT_MAX) { 556 return -1; 557 } 558 559 *result = (int)value; 560 return 0; 561 } 562 563 564 void _Py_get_env_flag(int use_environment,int * flag,const char * name)565 _Py_get_env_flag(int use_environment, int *flag, const char *name) 566 { 567 const char *var = _Py_GetEnv(use_environment, name); 568 if (!var) { 569 return; 570 } 571 int value; 572 if (_Py_str_to_int(var, &value) < 0 || value < 0) { 573 /* PYTHONDEBUG=text and PYTHONDEBUG=-2 behave as PYTHONDEBUG=1 */ 574 value = 1; 575 } 576 if (*flag < value) { 577 *flag = value; 578 } 579 } 580 581 582 const wchar_t* _Py_get_xoption(const PyWideStringList * xoptions,const wchar_t * name)583 _Py_get_xoption(const PyWideStringList *xoptions, const wchar_t *name) 584 { 585 for (Py_ssize_t i=0; i < xoptions->length; i++) { 586 const wchar_t *option = xoptions->items[i]; 587 size_t len; 588 wchar_t *sep = wcschr(option, L'='); 589 if (sep != NULL) { 590 len = (sep - option); 591 } 592 else { 593 len = wcslen(option); 594 } 595 if (wcsncmp(option, name, len) == 0 && name[len] == L'\0') { 596 return option; 597 } 598 } 599 return NULL; 600 } 601 602 603 static PyStatus preconfig_init_utf8_mode(PyPreConfig * config,const _PyPreCmdline * cmdline)604 preconfig_init_utf8_mode(PyPreConfig *config, const _PyPreCmdline *cmdline) 605 { 606 #ifdef MS_WINDOWS 607 if (config->legacy_windows_fs_encoding) { 608 config->utf8_mode = 0; 609 } 610 #endif 611 612 if (config->utf8_mode >= 0) { 613 return _PyStatus_OK(); 614 } 615 616 const wchar_t *xopt; 617 xopt = _Py_get_xoption(&cmdline->xoptions, L"utf8"); 618 if (xopt) { 619 wchar_t *sep = wcschr(xopt, L'='); 620 if (sep) { 621 xopt = sep + 1; 622 if (wcscmp(xopt, L"1") == 0) { 623 config->utf8_mode = 1; 624 } 625 else if (wcscmp(xopt, L"0") == 0) { 626 config->utf8_mode = 0; 627 } 628 else { 629 return _PyStatus_ERR("invalid -X utf8 option value"); 630 } 631 } 632 else { 633 config->utf8_mode = 1; 634 } 635 return _PyStatus_OK(); 636 } 637 638 const char *opt = _Py_GetEnv(config->use_environment, "PYTHONUTF8"); 639 if (opt) { 640 if (strcmp(opt, "1") == 0) { 641 config->utf8_mode = 1; 642 } 643 else if (strcmp(opt, "0") == 0) { 644 config->utf8_mode = 0; 645 } 646 else { 647 return _PyStatus_ERR("invalid PYTHONUTF8 environment " 648 "variable value"); 649 } 650 return _PyStatus_OK(); 651 } 652 653 654 #ifndef MS_WINDOWS 655 if (config->utf8_mode < 0) { 656 /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ 657 const char *ctype_loc = setlocale(LC_CTYPE, NULL); 658 if (ctype_loc != NULL 659 && (strcmp(ctype_loc, "C") == 0 660 || strcmp(ctype_loc, "POSIX") == 0)) 661 { 662 config->utf8_mode = 1; 663 } 664 } 665 #endif 666 667 if (config->utf8_mode < 0) { 668 config->utf8_mode = 0; 669 } 670 return _PyStatus_OK(); 671 } 672 673 674 static void preconfig_init_coerce_c_locale(PyPreConfig * config)675 preconfig_init_coerce_c_locale(PyPreConfig *config) 676 { 677 if (!config->configure_locale) { 678 config->coerce_c_locale = 0; 679 config->coerce_c_locale_warn = 0; 680 return; 681 } 682 683 const char *env = _Py_GetEnv(config->use_environment, "PYTHONCOERCECLOCALE"); 684 if (env) { 685 if (strcmp(env, "0") == 0) { 686 if (config->coerce_c_locale < 0) { 687 config->coerce_c_locale = 0; 688 } 689 } 690 else if (strcmp(env, "warn") == 0) { 691 if (config->coerce_c_locale_warn < 0) { 692 config->coerce_c_locale_warn = 1; 693 } 694 } 695 else { 696 if (config->coerce_c_locale < 0) { 697 config->coerce_c_locale = 1; 698 } 699 } 700 } 701 702 /* Test if coerce_c_locale equals to -1 or equals to 1: 703 PYTHONCOERCECLOCALE=1 doesn't imply that the C locale is always coerced. 704 It is only coerced if if the LC_CTYPE locale is "C". */ 705 if (config->coerce_c_locale < 0 || config->coerce_c_locale == 1) { 706 /* The C locale enables the C locale coercion (PEP 538) */ 707 if (_Py_LegacyLocaleDetected(0)) { 708 config->coerce_c_locale = 2; 709 } 710 else { 711 config->coerce_c_locale = 0; 712 } 713 } 714 715 if (config->coerce_c_locale_warn < 0) { 716 config->coerce_c_locale_warn = 0; 717 } 718 } 719 720 721 static PyStatus preconfig_init_allocator(PyPreConfig * config)722 preconfig_init_allocator(PyPreConfig *config) 723 { 724 if (config->allocator == PYMEM_ALLOCATOR_NOT_SET) { 725 /* bpo-34247. The PYTHONMALLOC environment variable has the priority 726 over PYTHONDEV env var and "-X dev" command line option. 727 For example, PYTHONMALLOC=malloc PYTHONDEVMODE=1 sets the memory 728 allocators to "malloc" (and not to "debug"). */ 729 const char *envvar = _Py_GetEnv(config->use_environment, "PYTHONMALLOC"); 730 if (envvar) { 731 PyMemAllocatorName name; 732 if (_PyMem_GetAllocatorName(envvar, &name) < 0) { 733 return _PyStatus_ERR("PYTHONMALLOC: unknown allocator"); 734 } 735 config->allocator = (int)name; 736 } 737 } 738 739 if (config->dev_mode && config->allocator == PYMEM_ALLOCATOR_NOT_SET) { 740 config->allocator = PYMEM_ALLOCATOR_DEBUG; 741 } 742 return _PyStatus_OK(); 743 } 744 745 746 static PyStatus preconfig_read(PyPreConfig * config,_PyPreCmdline * cmdline)747 preconfig_read(PyPreConfig *config, _PyPreCmdline *cmdline) 748 { 749 PyStatus status; 750 751 status = _PyPreCmdline_Read(cmdline, config); 752 if (_PyStatus_EXCEPTION(status)) { 753 return status; 754 } 755 756 precmdline_set_preconfig(cmdline, config); 757 758 /* legacy_windows_fs_encoding, coerce_c_locale, utf8_mode */ 759 #ifdef MS_WINDOWS 760 _Py_get_env_flag(config->use_environment, 761 &config->legacy_windows_fs_encoding, 762 "PYTHONLEGACYWINDOWSFSENCODING"); 763 #endif 764 765 preconfig_init_coerce_c_locale(config); 766 767 status = preconfig_init_utf8_mode(config, cmdline); 768 if (_PyStatus_EXCEPTION(status)) { 769 return status; 770 } 771 772 /* allocator */ 773 status = preconfig_init_allocator(config); 774 if (_PyStatus_EXCEPTION(status)) { 775 return status; 776 } 777 778 assert(config->coerce_c_locale >= 0); 779 assert(config->coerce_c_locale_warn >= 0); 780 #ifdef MS_WINDOWS 781 assert(config->legacy_windows_fs_encoding >= 0); 782 #endif 783 assert(config->utf8_mode >= 0); 784 assert(config->isolated >= 0); 785 assert(config->use_environment >= 0); 786 assert(config->dev_mode >= 0); 787 788 return _PyStatus_OK(); 789 } 790 791 792 /* Read the configuration from: 793 794 - command line arguments 795 - environment variables 796 - Py_xxx global configuration variables 797 - the LC_CTYPE locale */ 798 PyStatus _PyPreConfig_Read(PyPreConfig * config,const _PyArgv * args)799 _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) 800 { 801 PyStatus status; 802 803 status = _PyRuntime_Initialize(); 804 if (_PyStatus_EXCEPTION(status)) { 805 return status; 806 } 807 808 preconfig_get_global_vars(config); 809 810 /* Copy LC_CTYPE locale, since it's modified later */ 811 const char *loc = setlocale(LC_CTYPE, NULL); 812 if (loc == NULL) { 813 return _PyStatus_ERR("failed to LC_CTYPE locale"); 814 } 815 char *init_ctype_locale = _PyMem_RawStrdup(loc); 816 if (init_ctype_locale == NULL) { 817 return _PyStatus_NO_MEMORY(); 818 } 819 820 /* Save the config to be able to restore it if encodings change */ 821 PyPreConfig save_config; 822 823 status = _PyPreConfig_InitFromPreConfig(&save_config, config); 824 if (_PyStatus_EXCEPTION(status)) { 825 return status; 826 } 827 828 /* Set LC_CTYPE to the user preferred locale */ 829 if (config->configure_locale) { 830 _Py_SetLocaleFromEnv(LC_CTYPE); 831 } 832 833 _PyPreCmdline cmdline = _PyPreCmdline_INIT; 834 int init_utf8_mode = Py_UTF8Mode; 835 #ifdef MS_WINDOWS 836 int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag; 837 #endif 838 839 int locale_coerced = 0; 840 int loops = 0; 841 842 while (1) { 843 int utf8_mode = config->utf8_mode; 844 845 /* Watchdog to prevent an infinite loop */ 846 loops++; 847 if (loops == 3) { 848 status = _PyStatus_ERR("Encoding changed twice while " 849 "reading the configuration"); 850 goto done; 851 } 852 853 /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend 854 on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */ 855 Py_UTF8Mode = config->utf8_mode; 856 #ifdef MS_WINDOWS 857 Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; 858 #endif 859 860 if (args) { 861 // Set command line arguments at each iteration. If they are bytes 862 // strings, they are decoded from the new encoding. 863 status = _PyPreCmdline_SetArgv(&cmdline, args); 864 if (_PyStatus_EXCEPTION(status)) { 865 goto done; 866 } 867 } 868 869 status = preconfig_read(config, &cmdline); 870 if (_PyStatus_EXCEPTION(status)) { 871 goto done; 872 } 873 874 /* The legacy C locale assumes ASCII as the default text encoding, which 875 * causes problems not only for the CPython runtime, but also other 876 * components like GNU readline. 877 * 878 * Accordingly, when the CLI detects it, it attempts to coerce it to a 879 * more capable UTF-8 based alternative. 880 * 881 * See the documentation of the PYTHONCOERCECLOCALE setting for more 882 * details. 883 */ 884 int encoding_changed = 0; 885 if (config->coerce_c_locale && !locale_coerced) { 886 locale_coerced = 1; 887 _Py_CoerceLegacyLocale(0); 888 encoding_changed = 1; 889 } 890 891 if (utf8_mode == -1) { 892 if (config->utf8_mode == 1) { 893 /* UTF-8 Mode enabled */ 894 encoding_changed = 1; 895 } 896 } 897 else { 898 if (config->utf8_mode != utf8_mode) { 899 encoding_changed = 1; 900 } 901 } 902 903 if (!encoding_changed) { 904 break; 905 } 906 907 /* Reset the configuration before reading again the configuration, 908 just keep UTF-8 Mode and coerce C locale value. */ 909 int new_utf8_mode = config->utf8_mode; 910 int new_coerce_c_locale = config->coerce_c_locale; 911 preconfig_copy(config, &save_config); 912 config->utf8_mode = new_utf8_mode; 913 config->coerce_c_locale = new_coerce_c_locale; 914 915 /* The encoding changed: read again the configuration 916 with the new encoding */ 917 } 918 status = _PyStatus_OK(); 919 920 done: 921 if (init_ctype_locale != NULL) { 922 setlocale(LC_CTYPE, init_ctype_locale); 923 PyMem_RawFree(init_ctype_locale); 924 } 925 Py_UTF8Mode = init_utf8_mode ; 926 #ifdef MS_WINDOWS 927 Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding; 928 #endif 929 _PyPreCmdline_Clear(&cmdline); 930 return status; 931 } 932 933 934 /* Write the pre-configuration: 935 936 - set the memory allocators 937 - set Py_xxx global configuration variables 938 - set the LC_CTYPE locale (coerce C locale, PEP 538) and set the UTF-8 mode 939 (PEP 540) 940 941 The applied configuration is written into _PyRuntime.preconfig. 942 If the C locale cannot be coerced, set coerce_c_locale to 0. 943 944 Do nothing if called after Py_Initialize(): ignore the new 945 pre-configuration. */ 946 PyStatus _PyPreConfig_Write(const PyPreConfig * src_config)947 _PyPreConfig_Write(const PyPreConfig *src_config) 948 { 949 PyPreConfig config; 950 951 PyStatus status = _PyPreConfig_InitFromPreConfig(&config, src_config); 952 if (_PyStatus_EXCEPTION(status)) { 953 return status; 954 } 955 956 if (_PyRuntime.core_initialized) { 957 /* bpo-34008: Calling this functions after Py_Initialize() ignores 958 the new configuration. */ 959 return _PyStatus_OK(); 960 } 961 962 PyMemAllocatorName name = (PyMemAllocatorName)config.allocator; 963 if (name != PYMEM_ALLOCATOR_NOT_SET) { 964 if (_PyMem_SetupAllocators(name) < 0) { 965 return _PyStatus_ERR("Unknown PYTHONMALLOC allocator"); 966 } 967 } 968 969 preconfig_set_global_vars(&config); 970 971 if (config.configure_locale) { 972 if (config.coerce_c_locale) { 973 if (!_Py_CoerceLegacyLocale(config.coerce_c_locale_warn)) { 974 /* C locale not coerced */ 975 config.coerce_c_locale = 0; 976 } 977 } 978 979 /* Set LC_CTYPE to the user preferred locale */ 980 _Py_SetLocaleFromEnv(LC_CTYPE); 981 } 982 983 /* Write the new pre-configuration into _PyRuntime */ 984 preconfig_copy(&_PyRuntime.preconfig, &config); 985 986 return _PyStatus_OK(); 987 } 988