1# Copyright (C) 2001-2006 Python Software Foundation 2# Author: Ben Gertzfield, Barry Warsaw 3# Contact: email-sig@python.org 4 5__all__ = [ 6 'Charset', 7 'add_alias', 8 'add_charset', 9 'add_codec', 10 ] 11 12import codecs 13import email.base64mime 14import email.quoprimime 15 16from email import errors 17from email.encoders import encode_7or8bit 18 19 20 21# Flags for types of header encodings 22QP = 1 # Quoted-Printable 23BASE64 = 2 # Base64 24SHORTEST = 3 # the shorter of QP and base64, but only for headers 25 26# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 27MISC_LEN = 7 28 29DEFAULT_CHARSET = 'us-ascii' 30 31 32 33# Defaults 34CHARSETS = { 35 # input header enc body enc output conv 36 'iso-8859-1': (QP, QP, None), 37 'iso-8859-2': (QP, QP, None), 38 'iso-8859-3': (QP, QP, None), 39 'iso-8859-4': (QP, QP, None), 40 # iso-8859-5 is Cyrillic, and not especially used 41 # iso-8859-6 is Arabic, also not particularly used 42 # iso-8859-7 is Greek, QP will not make it readable 43 # iso-8859-8 is Hebrew, QP will not make it readable 44 'iso-8859-9': (QP, QP, None), 45 'iso-8859-10': (QP, QP, None), 46 # iso-8859-11 is Thai, QP will not make it readable 47 'iso-8859-13': (QP, QP, None), 48 'iso-8859-14': (QP, QP, None), 49 'iso-8859-15': (QP, QP, None), 50 'iso-8859-16': (QP, QP, None), 51 'windows-1252':(QP, QP, None), 52 'viscii': (QP, QP, None), 53 'us-ascii': (None, None, None), 54 'big5': (BASE64, BASE64, None), 55 'gb2312': (BASE64, BASE64, None), 56 'euc-jp': (BASE64, None, 'iso-2022-jp'), 57 'shift_jis': (BASE64, None, 'iso-2022-jp'), 58 'iso-2022-jp': (BASE64, None, None), 59 'koi8-r': (BASE64, BASE64, None), 60 'utf-8': (SHORTEST, BASE64, 'utf-8'), 61 # We're making this one up to represent raw unencoded 8-bit 62 '8bit': (None, BASE64, 'utf-8'), 63 } 64 65# Aliases for other commonly-used names for character sets. Map 66# them to the real ones used in email. 67ALIASES = { 68 'latin_1': 'iso-8859-1', 69 'latin-1': 'iso-8859-1', 70 'latin_2': 'iso-8859-2', 71 'latin-2': 'iso-8859-2', 72 'latin_3': 'iso-8859-3', 73 'latin-3': 'iso-8859-3', 74 'latin_4': 'iso-8859-4', 75 'latin-4': 'iso-8859-4', 76 'latin_5': 'iso-8859-9', 77 'latin-5': 'iso-8859-9', 78 'latin_6': 'iso-8859-10', 79 'latin-6': 'iso-8859-10', 80 'latin_7': 'iso-8859-13', 81 'latin-7': 'iso-8859-13', 82 'latin_8': 'iso-8859-14', 83 'latin-8': 'iso-8859-14', 84 'latin_9': 'iso-8859-15', 85 'latin-9': 'iso-8859-15', 86 'latin_10':'iso-8859-16', 87 'latin-10':'iso-8859-16', 88 'cp949': 'ks_c_5601-1987', 89 'euc_jp': 'euc-jp', 90 'euc_kr': 'euc-kr', 91 'ascii': 'us-ascii', 92 } 93 94 95# Map charsets to their Unicode codec strings. 96CODEC_MAP = { 97 'gb2312': 'eucgb2312_cn', 98 'big5': 'big5_tw', 99 # Hack: We don't want *any* conversion for stuff marked us-ascii, as all 100 # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. 101 # Let that stuff pass through without conversion to/from Unicode. 102 'us-ascii': None, 103 } 104 105 106 107# Convenience functions for extending the above mappings 108def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): 109 """Add character set properties to the global registry. 110 111 charset is the input character set, and must be the canonical name of a 112 character set. 113 114 Optional header_enc and body_enc is either Charset.QP for 115 quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for 116 the shortest of qp or base64 encoding, or None for no encoding. SHORTEST 117 is only valid for header_enc. It describes how message headers and 118 message bodies in the input charset are to be encoded. Default is no 119 encoding. 120 121 Optional output_charset is the character set that the output should be 122 in. Conversions will proceed from input charset, to Unicode, to the 123 output charset when the method Charset.convert() is called. The default 124 is to output in the same character set as the input. 125 126 Both input_charset and output_charset must have Unicode codec entries in 127 the module's charset-to-codec mapping; use add_codec(charset, codecname) 128 to add codecs the module does not know about. See the codecs module's 129 documentation for more information. 130 """ 131 if body_enc == SHORTEST: 132 raise ValueError('SHORTEST not allowed for body_enc') 133 CHARSETS[charset] = (header_enc, body_enc, output_charset) 134 135 136def add_alias(alias, canonical): 137 """Add a character set alias. 138 139 alias is the alias name, e.g. latin-1 140 canonical is the character set's canonical name, e.g. iso-8859-1 141 """ 142 ALIASES[alias] = canonical 143 144 145def add_codec(charset, codecname): 146 """Add a codec that map characters in the given charset to/from Unicode. 147 148 charset is the canonical name of a character set. codecname is the name 149 of a Python codec, as appropriate for the second argument to the unicode() 150 built-in, or to the encode() method of a Unicode string. 151 """ 152 CODEC_MAP[charset] = codecname 153 154 155 156class Charset: 157 """Map character sets to their email properties. 158 159 This class provides information about the requirements imposed on email 160 for a specific character set. It also provides convenience routines for 161 converting between character sets, given the availability of the 162 applicable codecs. Given a character set, it will do its best to provide 163 information on how to use that character set in an email in an 164 RFC-compliant way. 165 166 Certain character sets must be encoded with quoted-printable or base64 167 when used in email headers or bodies. Certain character sets must be 168 converted outright, and are not allowed in email. Instances of this 169 module expose the following information about a character set: 170 171 input_charset: The initial character set specified. Common aliases 172 are converted to their `official' email names (e.g. latin_1 173 is converted to iso-8859-1). Defaults to 7-bit us-ascii. 174 175 header_encoding: If the character set must be encoded before it can be 176 used in an email header, this attribute will be set to 177 Charset.QP (for quoted-printable), Charset.BASE64 (for 178 base64 encoding), or Charset.SHORTEST for the shortest of 179 QP or BASE64 encoding. Otherwise, it will be None. 180 181 body_encoding: Same as header_encoding, but describes the encoding for the 182 mail message's body, which indeed may be different than the 183 header encoding. Charset.SHORTEST is not allowed for 184 body_encoding. 185 186 output_charset: Some character sets must be converted before they can be 187 used in email headers or bodies. If the input_charset is 188 one of them, this attribute will contain the name of the 189 charset output will be converted to. Otherwise, it will 190 be None. 191 192 input_codec: The name of the Python codec used to convert the 193 input_charset to Unicode. If no conversion codec is 194 necessary, this attribute will be None. 195 196 output_codec: The name of the Python codec used to convert Unicode 197 to the output_charset. If no conversion codec is necessary, 198 this attribute will have the same value as the input_codec. 199 """ 200 def __init__(self, input_charset=DEFAULT_CHARSET): 201 # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to 202 # unicode because its .lower() is locale insensitive. If the argument 203 # is already a unicode, we leave it at that, but ensure that the 204 # charset is ASCII, as the standard (RFC XXX) requires. 205 try: 206 if isinstance(input_charset, unicode): 207 input_charset.encode('ascii') 208 else: 209 input_charset = unicode(input_charset, 'ascii') 210 except UnicodeError: 211 raise errors.CharsetError(input_charset) 212 input_charset = input_charset.lower().encode('ascii') 213 # Set the input charset after filtering through the aliases and/or codecs 214 if not (input_charset in ALIASES or input_charset in CHARSETS): 215 try: 216 input_charset = codecs.lookup(input_charset).name 217 except LookupError: 218 pass 219 self.input_charset = ALIASES.get(input_charset, input_charset) 220 # We can try to guess which encoding and conversion to use by the 221 # charset_map dictionary. Try that first, but let the user override 222 # it. 223 henc, benc, conv = CHARSETS.get(self.input_charset, 224 (SHORTEST, BASE64, None)) 225 if not conv: 226 conv = self.input_charset 227 # Set the attributes, allowing the arguments to override the default. 228 self.header_encoding = henc 229 self.body_encoding = benc 230 self.output_charset = ALIASES.get(conv, conv) 231 # Now set the codecs. If one isn't defined for input_charset, 232 # guess and try a Unicode codec with the same name as input_codec. 233 self.input_codec = CODEC_MAP.get(self.input_charset, 234 self.input_charset) 235 self.output_codec = CODEC_MAP.get(self.output_charset, 236 self.output_charset) 237 238 def __str__(self): 239 return self.input_charset.lower() 240 241 __repr__ = __str__ 242 243 def __eq__(self, other): 244 return str(self) == str(other).lower() 245 246 def __ne__(self, other): 247 return not self.__eq__(other) 248 249 def get_body_encoding(self): 250 """Return the content-transfer-encoding used for body encoding. 251 252 This is either the string `quoted-printable' or `base64' depending on 253 the encoding used, or it is a function in which case you should call 254 the function with a single argument, the Message object being 255 encoded. The function should then set the Content-Transfer-Encoding 256 header itself to whatever is appropriate. 257 258 Returns "quoted-printable" if self.body_encoding is QP. 259 Returns "base64" if self.body_encoding is BASE64. 260 Returns "7bit" otherwise. 261 """ 262 assert self.body_encoding != SHORTEST 263 if self.body_encoding == QP: 264 return 'quoted-printable' 265 elif self.body_encoding == BASE64: 266 return 'base64' 267 else: 268 return encode_7or8bit 269 270 def convert(self, s): 271 """Convert a string from the input_codec to the output_codec.""" 272 if self.input_codec != self.output_codec: 273 return unicode(s, self.input_codec).encode(self.output_codec) 274 else: 275 return s 276 277 def to_splittable(self, s): 278 """Convert a possibly multibyte string to a safely splittable format. 279 280 Uses the input_codec to try and convert the string to Unicode, so it 281 can be safely split on character boundaries (even for multibyte 282 characters). 283 284 Returns the string as-is if it isn't known how to convert it to 285 Unicode with the input_charset. 286 287 Characters that could not be converted to Unicode will be replaced 288 with the Unicode replacement character U+FFFD. 289 """ 290 if isinstance(s, unicode) or self.input_codec is None: 291 return s 292 try: 293 return unicode(s, self.input_codec, 'replace') 294 except LookupError: 295 # Input codec not installed on system, so return the original 296 # string unchanged. 297 return s 298 299 def from_splittable(self, ustr, to_output=True): 300 """Convert a splittable string back into an encoded string. 301 302 Uses the proper codec to try and convert the string from Unicode back 303 into an encoded format. Return the string as-is if it is not Unicode, 304 or if it could not be converted from Unicode. 305 306 Characters that could not be converted from Unicode will be replaced 307 with an appropriate character (usually '?'). 308 309 If to_output is True (the default), uses output_codec to convert to an 310 encoded format. If to_output is False, uses input_codec. 311 """ 312 if to_output: 313 codec = self.output_codec 314 else: 315 codec = self.input_codec 316 if not isinstance(ustr, unicode) or codec is None: 317 return ustr 318 try: 319 return ustr.encode(codec, 'replace') 320 except LookupError: 321 # Output codec not installed 322 return ustr 323 324 def get_output_charset(self): 325 """Return the output character set. 326 327 This is self.output_charset if that is not None, otherwise it is 328 self.input_charset. 329 """ 330 return self.output_charset or self.input_charset 331 332 def encoded_header_len(self, s): 333 """Return the length of the encoded header string.""" 334 cset = self.get_output_charset() 335 # The len(s) of a 7bit encoding is len(s) 336 if self.header_encoding == BASE64: 337 return email.base64mime.base64_len(s) + len(cset) + MISC_LEN 338 elif self.header_encoding == QP: 339 return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN 340 elif self.header_encoding == SHORTEST: 341 lenb64 = email.base64mime.base64_len(s) 342 lenqp = email.quoprimime.header_quopri_len(s) 343 return min(lenb64, lenqp) + len(cset) + MISC_LEN 344 else: 345 return len(s) 346 347 def header_encode(self, s, convert=False): 348 """Header-encode a string, optionally converting it to output_charset. 349 350 If convert is True, the string will be converted from the input 351 charset to the output charset automatically. This is not useful for 352 multibyte character sets, which have line length issues (multibyte 353 characters must be split on a character, not a byte boundary); use the 354 high-level Header class to deal with these issues. convert defaults 355 to False. 356 357 The type of encoding (base64 or quoted-printable) will be based on 358 self.header_encoding. 359 """ 360 cset = self.get_output_charset() 361 if convert: 362 s = self.convert(s) 363 # 7bit/8bit encodings return the string unchanged (modulo conversions) 364 if self.header_encoding == BASE64: 365 return email.base64mime.header_encode(s, cset) 366 elif self.header_encoding == QP: 367 return email.quoprimime.header_encode(s, cset, maxlinelen=None) 368 elif self.header_encoding == SHORTEST: 369 lenb64 = email.base64mime.base64_len(s) 370 lenqp = email.quoprimime.header_quopri_len(s) 371 if lenb64 < lenqp: 372 return email.base64mime.header_encode(s, cset) 373 else: 374 return email.quoprimime.header_encode(s, cset, maxlinelen=None) 375 else: 376 return s 377 378 def body_encode(self, s, convert=True): 379 """Body-encode a string and convert it to output_charset. 380 381 If convert is True (the default), the string will be converted from 382 the input charset to output charset automatically. Unlike 383 header_encode(), there are no issues with byte boundaries and 384 multibyte charsets in email bodies, so this is usually pretty safe. 385 386 The type of encoding (base64 or quoted-printable) will be based on 387 self.body_encoding. 388 """ 389 if convert: 390 s = self.convert(s) 391 # 7bit/8bit encodings return the string unchanged (module conversions) 392 if self.body_encoding is BASE64: 393 return email.base64mime.body_encode(s) 394 elif self.body_encoding is QP: 395 return email.quoprimime.body_encode(s) 396 else: 397 return s 398