1 """ codecs -- Python Codec Registry, API and helpers.
4 Written by Marc-Andre Lemburg (mal@lemburg.com).
36 # Byte Order Mark (BOM = ZERO WIDTH NO-BREAK SPACE = U+FEFF)
41 # UTF-8
44 # UTF-16, little endian
47 # UTF-16, big endian
50 # UTF-32, little endian
53 # UTF-32, big endian
58     # UTF-16, native endianness
61     # UTF-32, native endianness
66     # UTF-16, native endianness
69     # UTF-32, native endianness
84     # Private API to allow Python to blacklist the known non-Unicode
115         handling schemes by providing the errors argument. These
118          'strict' - raise a ValueError error (or a subclass)
119          'ignore' - ignore the character and continue with the next
120          'replace' - replace with a suitable replacement character;
124          'xmlcharrefreplace' - Replace with the appropriate XML
126          'backslashreplace'  - Replace with backslashed escape sequences
132     def encode(self, input, errors='strict'):  argument
137             errors defines the error handling to apply. It defaults to
151     def decode(self, input, errors='strict'):  argument
160             errors defines the error handling to apply. It defaults to
180     def __init__(self, errors='strict'):  argument
185         providing the errors keyword argument. See the module docstring
188         self.errors = errors
220     def __init__(self, errors='strict'):  argument
221         IncrementalEncoder.__init__(self, errors)
224     def _buffer_encode(self, input, errors, final):  argument
232         (result, consumed) = self._buffer_encode(data, self.errors, final)
253     def __init__(self, errors='strict'):  argument
258         providing the errors keyword argument. See the module docstring
261         self.errors = errors
281         additional_state_info must be a non-negative integer
302     def __init__(self, errors='strict'):  argument
303         IncrementalDecoder.__init__(self, errors)
306     def _buffer_decode(self, input, errors, final):  argument
314         (result, consumed) = self._buffer_decode(data, self.errors, final)
328         # ignore additional state info
340     def __init__(self, stream, errors='strict'):  argument
344             stream must be a file-like object open for writing
348             schemes by providing the errors keyword argument. These
351              'strict' - raise a ValueError (or a subclass)
352              'ignore' - ignore the character and continue with the next
353              'replace'- replace with a suitable replacement character
354              'xmlcharrefreplace' - Replace with the appropriate XML
356              'backslashreplace'  - Replace with backslashed escape
363         self.errors = errors
369         data, consumed = self.encode(object, self.errors)
413     def __init__(self, stream, errors='strict'):  argument
417             stream must be a file-like object open for reading
421             schemes by providing the errors keyword argument. These
424              'strict' - raise a ValueError (or a subclass)
425              'ignore' - ignore the character and continue with the next
426              'replace'- replace with a suitable replacement character;
432         self.errors = errors
434         # For str->str decoding this will stay a str
435         # For str->unicode decoding the first read will promote it to unicode
439     def decode(self, input, errors='strict'):  argument
442     def read(self, size=-1, chars=-1, firstline=False):
455             -1 indicates to read and decode as much as possible.  size
494                 newchars, decodedbytes = self.decode(data, self.errors)
497                     newchars, decodedbytes = self.decode(data[:exc.start], self.errors)
565                         lines[-1] += self.charbuffer
602             way to finding the true end-of-line.
614             from decoding errors.
668     def __init__(self, stream, Reader, Writer, errors='strict'):  argument
672             stream must be a Stream-like object.
682         self.reader = Reader(stream, errors)
683         self.writer = Writer(stream, errors)
684         self.errors = errors
686     def read(self, size=-1):
744     """ StreamRecoder instances provide a frontend - backend
764                  errors='strict'):  argument
766         """ Creates a StreamRecoder instance which implements a two-way
773             recodings from e.g. latin-1 to utf-8 and back.
775             stream must be a file-like object.
792         self.reader = Reader(stream, errors)
793         self.writer = Writer(stream, errors)
794         self.errors = errors
796     def read(self, size=-1):
799         data, bytesencoded = self.encode(data, self.errors)
808         data, bytesencoded = self.encode(data, self.errors)
814         data, bytesencoded = self.encode(data, self.errors)
821         data, bytesencoded = self.encode(data, self.errors)
829         data, bytesdecoded = self.decode(data, self.errors)
835         data, bytesdecoded = self.decode(data, self.errors)
858 def open(filename, mode='rb', encoding=None, errors='strict', buffering=1):  argument
870         using 8-bit values. The default file mode is 'rb' meaning to
876         errors may be given to define the error handling. It defaults
902     srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors)
907 def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'):  argument
922         errors may be given to define the error handling. It defaults
937                        file_info.streamreader, file_info.streamwriter, errors)
1013 def iterencode(iterator, encoding, errors='strict', **kwargs):  argument
1019     errors and kwargs are passed through to the IncrementalEncoder
1022     encoder = getincrementalencoder(encoding)(errors, **kwargs)
1031 def iterdecode(iterator, encoding, errors='strict', **kwargs):  argument
1037     errors and kwargs are passed through to the IncrementalDecoder
1040     decoder = getincrementaldecoder(encoding)(errors, **kwargs)
1049 ### Helpers for charmap-based codecs
1053     """ make_identity_dict(rng) -> dict
1089     ignore_errors = lookup_error("ignore")
1094     # In --disable-unicode builds, these error handler are missing
1111     # Make stdout translate Latin-1 output into UTF-8 output
1112     sys.stdout = EncodedFile(sys.stdout, 'latin-1', 'utf-8')
1114     # Have stdin translate Latin-1 input into UTF-8 input
1115     sys.stdin = EncodedFile(sys.stdin, 'utf-8', 'latin-1')