• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2001-2006 Python Software Foundation
2# Author: Ben Gertzfield, Barry Warsaw
3# Contact: email-sig@python.org
4
5__all__ = [
6    'Charset',
7    'add_alias',
8    'add_charset',
9    'add_codec',
10    ]
11
12import codecs
13import email.base64mime
14import email.quoprimime
15
16from email import errors
17from email.encoders import encode_7or8bit
18
19
20
21# Flags for types of header encodings
22QP          = 1 # Quoted-Printable
23BASE64      = 2 # Base64
24SHORTEST    = 3 # the shorter of QP and base64, but only for headers
25
26# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
27MISC_LEN = 7
28
29DEFAULT_CHARSET = 'us-ascii'
30
31
32
33# Defaults
34CHARSETS = {
35    # input        header enc  body enc output conv
36    'iso-8859-1':  (QP,        QP,      None),
37    'iso-8859-2':  (QP,        QP,      None),
38    'iso-8859-3':  (QP,        QP,      None),
39    'iso-8859-4':  (QP,        QP,      None),
40    # iso-8859-5 is Cyrillic, and not especially used
41    # iso-8859-6 is Arabic, also not particularly used
42    # iso-8859-7 is Greek, QP will not make it readable
43    # iso-8859-8 is Hebrew, QP will not make it readable
44    'iso-8859-9':  (QP,        QP,      None),
45    'iso-8859-10': (QP,        QP,      None),
46    # iso-8859-11 is Thai, QP will not make it readable
47    'iso-8859-13': (QP,        QP,      None),
48    'iso-8859-14': (QP,        QP,      None),
49    'iso-8859-15': (QP,        QP,      None),
50    'iso-8859-16': (QP,        QP,      None),
51    'windows-1252':(QP,        QP,      None),
52    'viscii':      (QP,        QP,      None),
53    'us-ascii':    (None,      None,    None),
54    'big5':        (BASE64,    BASE64,  None),
55    'gb2312':      (BASE64,    BASE64,  None),
56    'euc-jp':      (BASE64,    None,    'iso-2022-jp'),
57    'shift_jis':   (BASE64,    None,    'iso-2022-jp'),
58    'iso-2022-jp': (BASE64,    None,    None),
59    'koi8-r':      (BASE64,    BASE64,  None),
60    'utf-8':       (SHORTEST,  BASE64, 'utf-8'),
61    # We're making this one up to represent raw unencoded 8-bit
62    '8bit':        (None,      BASE64, 'utf-8'),
63    }
64
65# Aliases for other commonly-used names for character sets.  Map
66# them to the real ones used in email.
67ALIASES = {
68    'latin_1': 'iso-8859-1',
69    'latin-1': 'iso-8859-1',
70    'latin_2': 'iso-8859-2',
71    'latin-2': 'iso-8859-2',
72    'latin_3': 'iso-8859-3',
73    'latin-3': 'iso-8859-3',
74    'latin_4': 'iso-8859-4',
75    'latin-4': 'iso-8859-4',
76    'latin_5': 'iso-8859-9',
77    'latin-5': 'iso-8859-9',
78    'latin_6': 'iso-8859-10',
79    'latin-6': 'iso-8859-10',
80    'latin_7': 'iso-8859-13',
81    'latin-7': 'iso-8859-13',
82    'latin_8': 'iso-8859-14',
83    'latin-8': 'iso-8859-14',
84    'latin_9': 'iso-8859-15',
85    'latin-9': 'iso-8859-15',
86    'latin_10':'iso-8859-16',
87    'latin-10':'iso-8859-16',
88    'cp949':   'ks_c_5601-1987',
89    'euc_jp':  'euc-jp',
90    'euc_kr':  'euc-kr',
91    'ascii':   'us-ascii',
92    }
93
94
95# Map charsets to their Unicode codec strings.
96CODEC_MAP = {
97    'gb2312':      'eucgb2312_cn',
98    'big5':        'big5_tw',
99    # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
100    # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
101    # Let that stuff pass through without conversion to/from Unicode.
102    'us-ascii':    None,
103    }
104
105
106
107# Convenience functions for extending the above mappings
108def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
109    """Add character set properties to the global registry.
110
111    charset is the input character set, and must be the canonical name of a
112    character set.
113
114    Optional header_enc and body_enc is either Charset.QP for
115    quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
116    the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
117    is only valid for header_enc.  It describes how message headers and
118    message bodies in the input charset are to be encoded.  Default is no
119    encoding.
120
121    Optional output_charset is the character set that the output should be
122    in.  Conversions will proceed from input charset, to Unicode, to the
123    output charset when the method Charset.convert() is called.  The default
124    is to output in the same character set as the input.
125
126    Both input_charset and output_charset must have Unicode codec entries in
127    the module's charset-to-codec mapping; use add_codec(charset, codecname)
128    to add codecs the module does not know about.  See the codecs module's
129    documentation for more information.
130    """
131    if body_enc == SHORTEST:
132        raise ValueError('SHORTEST not allowed for body_enc')
133    CHARSETS[charset] = (header_enc, body_enc, output_charset)
134
135
136def add_alias(alias, canonical):
137    """Add a character set alias.
138
139    alias is the alias name, e.g. latin-1
140    canonical is the character set's canonical name, e.g. iso-8859-1
141    """
142    ALIASES[alias] = canonical
143
144
145def add_codec(charset, codecname):
146    """Add a codec that map characters in the given charset to/from Unicode.
147
148    charset is the canonical name of a character set.  codecname is the name
149    of a Python codec, as appropriate for the second argument to the unicode()
150    built-in, or to the encode() method of a Unicode string.
151    """
152    CODEC_MAP[charset] = codecname
153
154
155
156class Charset:
157    """Map character sets to their email properties.
158
159    This class provides information about the requirements imposed on email
160    for a specific character set.  It also provides convenience routines for
161    converting between character sets, given the availability of the
162    applicable codecs.  Given a character set, it will do its best to provide
163    information on how to use that character set in an email in an
164    RFC-compliant way.
165
166    Certain character sets must be encoded with quoted-printable or base64
167    when used in email headers or bodies.  Certain character sets must be
168    converted outright, and are not allowed in email.  Instances of this
169    module expose the following information about a character set:
170
171    input_charset: The initial character set specified.  Common aliases
172                   are converted to their `official' email names (e.g. latin_1
173                   is converted to iso-8859-1).  Defaults to 7-bit us-ascii.
174
175    header_encoding: If the character set must be encoded before it can be
176                     used in an email header, this attribute will be set to
177                     Charset.QP (for quoted-printable), Charset.BASE64 (for
178                     base64 encoding), or Charset.SHORTEST for the shortest of
179                     QP or BASE64 encoding.  Otherwise, it will be None.
180
181    body_encoding: Same as header_encoding, but describes the encoding for the
182                   mail message's body, which indeed may be different than the
183                   header encoding.  Charset.SHORTEST is not allowed for
184                   body_encoding.
185
186    output_charset: Some character sets must be converted before they can be
187                    used in email headers or bodies.  If the input_charset is
188                    one of them, this attribute will contain the name of the
189                    charset output will be converted to.  Otherwise, it will
190                    be None.
191
192    input_codec: The name of the Python codec used to convert the
193                 input_charset to Unicode.  If no conversion codec is
194                 necessary, this attribute will be None.
195
196    output_codec: The name of the Python codec used to convert Unicode
197                  to the output_charset.  If no conversion codec is necessary,
198                  this attribute will have the same value as the input_codec.
199    """
200    def __init__(self, input_charset=DEFAULT_CHARSET):
201        # RFC 2046, $4.1.2 says charsets are not case sensitive.  We coerce to
202        # unicode because its .lower() is locale insensitive.  If the argument
203        # is already a unicode, we leave it at that, but ensure that the
204        # charset is ASCII, as the standard (RFC XXX) requires.
205        try:
206            if isinstance(input_charset, unicode):
207                input_charset.encode('ascii')
208            else:
209                input_charset = unicode(input_charset, 'ascii')
210        except UnicodeError:
211            raise errors.CharsetError(input_charset)
212        input_charset = input_charset.lower().encode('ascii')
213        # Set the input charset after filtering through the aliases and/or codecs
214        if not (input_charset in ALIASES or input_charset in CHARSETS):
215            try:
216                input_charset = codecs.lookup(input_charset).name
217            except LookupError:
218                pass
219        self.input_charset = ALIASES.get(input_charset, input_charset)
220        # We can try to guess which encoding and conversion to use by the
221        # charset_map dictionary.  Try that first, but let the user override
222        # it.
223        henc, benc, conv = CHARSETS.get(self.input_charset,
224                                        (SHORTEST, BASE64, None))
225        if not conv:
226            conv = self.input_charset
227        # Set the attributes, allowing the arguments to override the default.
228        self.header_encoding = henc
229        self.body_encoding = benc
230        self.output_charset = ALIASES.get(conv, conv)
231        # Now set the codecs.  If one isn't defined for input_charset,
232        # guess and try a Unicode codec with the same name as input_codec.
233        self.input_codec = CODEC_MAP.get(self.input_charset,
234                                         self.input_charset)
235        self.output_codec = CODEC_MAP.get(self.output_charset,
236                                          self.output_charset)
237
238    def __str__(self):
239        return self.input_charset.lower()
240
241    __repr__ = __str__
242
243    def __eq__(self, other):
244        return str(self) == str(other).lower()
245
246    def __ne__(self, other):
247        return not self.__eq__(other)
248
249    def get_body_encoding(self):
250        """Return the content-transfer-encoding used for body encoding.
251
252        This is either the string `quoted-printable' or `base64' depending on
253        the encoding used, or it is a function in which case you should call
254        the function with a single argument, the Message object being
255        encoded.  The function should then set the Content-Transfer-Encoding
256        header itself to whatever is appropriate.
257
258        Returns "quoted-printable" if self.body_encoding is QP.
259        Returns "base64" if self.body_encoding is BASE64.
260        Returns "7bit" otherwise.
261        """
262        assert self.body_encoding != SHORTEST
263        if self.body_encoding == QP:
264            return 'quoted-printable'
265        elif self.body_encoding == BASE64:
266            return 'base64'
267        else:
268            return encode_7or8bit
269
270    def convert(self, s):
271        """Convert a string from the input_codec to the output_codec."""
272        if self.input_codec != self.output_codec:
273            return unicode(s, self.input_codec).encode(self.output_codec)
274        else:
275            return s
276
277    def to_splittable(self, s):
278        """Convert a possibly multibyte string to a safely splittable format.
279
280        Uses the input_codec to try and convert the string to Unicode, so it
281        can be safely split on character boundaries (even for multibyte
282        characters).
283
284        Returns the string as-is if it isn't known how to convert it to
285        Unicode with the input_charset.
286
287        Characters that could not be converted to Unicode will be replaced
288        with the Unicode replacement character U+FFFD.
289        """
290        if isinstance(s, unicode) or self.input_codec is None:
291            return s
292        try:
293            return unicode(s, self.input_codec, 'replace')
294        except LookupError:
295            # Input codec not installed on system, so return the original
296            # string unchanged.
297            return s
298
299    def from_splittable(self, ustr, to_output=True):
300        """Convert a splittable string back into an encoded string.
301
302        Uses the proper codec to try and convert the string from Unicode back
303        into an encoded format.  Return the string as-is if it is not Unicode,
304        or if it could not be converted from Unicode.
305
306        Characters that could not be converted from Unicode will be replaced
307        with an appropriate character (usually '?').
308
309        If to_output is True (the default), uses output_codec to convert to an
310        encoded format.  If to_output is False, uses input_codec.
311        """
312        if to_output:
313            codec = self.output_codec
314        else:
315            codec = self.input_codec
316        if not isinstance(ustr, unicode) or codec is None:
317            return ustr
318        try:
319            return ustr.encode(codec, 'replace')
320        except LookupError:
321            # Output codec not installed
322            return ustr
323
324    def get_output_charset(self):
325        """Return the output character set.
326
327        This is self.output_charset if that is not None, otherwise it is
328        self.input_charset.
329        """
330        return self.output_charset or self.input_charset
331
332    def encoded_header_len(self, s):
333        """Return the length of the encoded header string."""
334        cset = self.get_output_charset()
335        # The len(s) of a 7bit encoding is len(s)
336        if self.header_encoding == BASE64:
337            return email.base64mime.base64_len(s) + len(cset) + MISC_LEN
338        elif self.header_encoding == QP:
339            return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN
340        elif self.header_encoding == SHORTEST:
341            lenb64 = email.base64mime.base64_len(s)
342            lenqp = email.quoprimime.header_quopri_len(s)
343            return min(lenb64, lenqp) + len(cset) + MISC_LEN
344        else:
345            return len(s)
346
347    def header_encode(self, s, convert=False):
348        """Header-encode a string, optionally converting it to output_charset.
349
350        If convert is True, the string will be converted from the input
351        charset to the output charset automatically.  This is not useful for
352        multibyte character sets, which have line length issues (multibyte
353        characters must be split on a character, not a byte boundary); use the
354        high-level Header class to deal with these issues.  convert defaults
355        to False.
356
357        The type of encoding (base64 or quoted-printable) will be based on
358        self.header_encoding.
359        """
360        cset = self.get_output_charset()
361        if convert:
362            s = self.convert(s)
363        # 7bit/8bit encodings return the string unchanged (modulo conversions)
364        if self.header_encoding == BASE64:
365            return email.base64mime.header_encode(s, cset)
366        elif self.header_encoding == QP:
367            return email.quoprimime.header_encode(s, cset, maxlinelen=None)
368        elif self.header_encoding == SHORTEST:
369            lenb64 = email.base64mime.base64_len(s)
370            lenqp = email.quoprimime.header_quopri_len(s)
371            if lenb64 < lenqp:
372                return email.base64mime.header_encode(s, cset)
373            else:
374                return email.quoprimime.header_encode(s, cset, maxlinelen=None)
375        else:
376            return s
377
378    def body_encode(self, s, convert=True):
379        """Body-encode a string and convert it to output_charset.
380
381        If convert is True (the default), the string will be converted from
382        the input charset to output charset automatically.  Unlike
383        header_encode(), there are no issues with byte boundaries and
384        multibyte charsets in email bodies, so this is usually pretty safe.
385
386        The type of encoding (base64 or quoted-printable) will be based on
387        self.body_encoding.
388        """
389        if convert:
390            s = self.convert(s)
391        # 7bit/8bit encodings return the string unchanged (module conversions)
392        if self.body_encoding is BASE64:
393            return email.base64mime.body_encode(s)
394        elif self.body_encoding is QP:
395            return email.quoprimime.body_encode(s)
396        else:
397            return s
398