1""" Routines for manipulating RFC2047 encoded words. 2 3This is currently a package-private API, but will be considered for promotion 4to a public API if there is demand. 5 6""" 7 8# An ecoded word looks like this: 9# 10# =?charset[*lang]?cte?encoded_string?= 11# 12# for more information about charset see the charset module. Here it is one 13# of the preferred MIME charset names (hopefully; you never know when parsing). 14# cte (Content Transfer Encoding) is either 'q' or 'b' (ignoring case). In 15# theory other letters could be used for other encodings, but in practice this 16# (almost?) never happens. There could be a public API for adding entries 17# to the CTE tables, but YAGNI for now. 'q' is Quoted Printable, 'b' is 18# Base64. The meaning of encoded_string should be obvious. 'lang' is optional 19# as indicated by the brackets (they are not part of the syntax) but is almost 20# never encountered in practice. 21# 22# The general interface for a CTE decoder is that it takes the encoded_string 23# as its argument, and returns a tuple (cte_decoded_string, defects). The 24# cte_decoded_string is the original binary that was encoded using the 25# specified cte. 'defects' is a list of MessageDefect instances indicating any 26# problems encountered during conversion. 'charset' and 'lang' are the 27# corresponding strings extracted from the EW, case preserved. 28# 29# The general interface for a CTE encoder is that it takes a binary sequence 30# as input and returns the cte_encoded_string, which is an ascii-only string. 31# 32# Each decoder must also supply a length function that takes the binary 33# sequence as its argument and returns the length of the resulting encoded 34# string. 35# 36# The main API functions for the module are decode, which calls the decoder 37# referenced by the cte specifier, and encode, which adds the appropriate 38# RFC 2047 "chrome" to the encoded string, and can optionally automatically 39# select the shortest possible encoding. See their docstrings below for 40# details. 41 42import re 43import base64 44import binascii 45import functools 46from string import ascii_letters, digits 47from email import errors 48 49__all__ = ['decode_q', 50 'encode_q', 51 'decode_b', 52 'encode_b', 53 'len_q', 54 'len_b', 55 'decode', 56 'encode', 57 ] 58 59# 60# Quoted Printable 61# 62 63# regex based decoder. 64_q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub, 65 lambda m: bytes([int(m.group(1), 16)])) 66 67def decode_q(encoded): 68 encoded = encoded.replace(b'_', b' ') 69 return _q_byte_subber(encoded), [] 70 71 72# dict mapping bytes to their encoded form 73class _QByteMap(dict): 74 75 safe = b'-!*+/' + ascii_letters.encode('ascii') + digits.encode('ascii') 76 77 def __missing__(self, key): 78 if key in self.safe: 79 self[key] = chr(key) 80 else: 81 self[key] = "={:02X}".format(key) 82 return self[key] 83 84_q_byte_map = _QByteMap() 85 86# In headers spaces are mapped to '_'. 87_q_byte_map[ord(' ')] = '_' 88 89def encode_q(bstring): 90 return ''.join(_q_byte_map[x] for x in bstring) 91 92def len_q(bstring): 93 return sum(len(_q_byte_map[x]) for x in bstring) 94 95 96# 97# Base64 98# 99 100def decode_b(encoded): 101 defects = [] 102 pad_err = len(encoded) % 4 103 if pad_err: 104 defects.append(errors.InvalidBase64PaddingDefect()) 105 padded_encoded = encoded + b'==='[:4-pad_err] 106 else: 107 padded_encoded = encoded 108 try: 109 return base64.b64decode(padded_encoded, validate=True), defects 110 except binascii.Error: 111 # Since we had correct padding, this must an invalid char error. 112 defects = [errors.InvalidBase64CharactersDefect()] 113 # The non-alphabet characters are ignored as far as padding 114 # goes, but we don't know how many there are. So we'll just 115 # try various padding lengths until something works. 116 for i in 0, 1, 2, 3: 117 try: 118 return base64.b64decode(encoded+b'='*i, validate=False), defects 119 except binascii.Error: 120 if i==0: 121 defects.append(errors.InvalidBase64PaddingDefect()) 122 else: 123 # This should never happen. 124 raise AssertionError("unexpected binascii.Error") 125 126def encode_b(bstring): 127 return base64.b64encode(bstring).decode('ascii') 128 129def len_b(bstring): 130 groups_of_3, leftover = divmod(len(bstring), 3) 131 # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. 132 return groups_of_3 * 4 + (4 if leftover else 0) 133 134 135_cte_decoders = { 136 'q': decode_q, 137 'b': decode_b, 138 } 139 140def decode(ew): 141 """Decode encoded word and return (string, charset, lang, defects) tuple. 142 143 An RFC 2047/2243 encoded word has the form: 144 145 =?charset*lang?cte?encoded_string?= 146 147 where '*lang' may be omitted but the other parts may not be. 148 149 This function expects exactly such a string (that is, it does not check the 150 syntax and may raise errors if the string is not well formed), and returns 151 the encoded_string decoded first from its Content Transfer Encoding and 152 then from the resulting bytes into unicode using the specified charset. If 153 the cte-decoded string does not successfully decode using the specified 154 character set, a defect is added to the defects list and the unknown octets 155 are replaced by the unicode 'unknown' character \\uFDFF. 156 157 The specified charset and language are returned. The default for language, 158 which is rarely if ever encountered, is the empty string. 159 160 """ 161 _, charset, cte, cte_string, _ = ew.split('?') 162 charset, _, lang = charset.partition('*') 163 cte = cte.lower() 164 # Recover the original bytes and do CTE decoding. 165 bstring = cte_string.encode('ascii', 'surrogateescape') 166 bstring, defects = _cte_decoders[cte](bstring) 167 # Turn the CTE decoded bytes into unicode. 168 try: 169 string = bstring.decode(charset) 170 except UnicodeError: 171 defects.append(errors.UndecodableBytesDefect("Encoded word " 172 "contains bytes not decodable using {} charset".format(charset))) 173 string = bstring.decode(charset, 'surrogateescape') 174 except LookupError: 175 string = bstring.decode('ascii', 'surrogateescape') 176 if charset.lower() != 'unknown-8bit': 177 defects.append(errors.CharsetError("Unknown charset {} " 178 "in encoded word; decoded as unknown bytes".format(charset))) 179 return string, charset, lang, defects 180 181 182_cte_encoders = { 183 'q': encode_q, 184 'b': encode_b, 185 } 186 187_cte_encode_length = { 188 'q': len_q, 189 'b': len_b, 190 } 191 192def encode(string, charset='utf-8', encoding=None, lang=''): 193 """Encode string using the CTE encoding that produces the shorter result. 194 195 Produces an RFC 2047/2243 encoded word of the form: 196 197 =?charset*lang?cte?encoded_string?= 198 199 where '*lang' is omitted unless the 'lang' parameter is given a value. 200 Optional argument charset (defaults to utf-8) specifies the charset to use 201 to encode the string to binary before CTE encoding it. Optional argument 202 'encoding' is the cte specifier for the encoding that should be used ('q' 203 or 'b'); if it is None (the default) the encoding which produces the 204 shortest encoded sequence is used, except that 'q' is preferred if it is up 205 to five characters longer. Optional argument 'lang' (default '') gives the 206 RFC 2243 language string to specify in the encoded word. 207 208 """ 209 if charset == 'unknown-8bit': 210 bstring = string.encode('ascii', 'surrogateescape') 211 else: 212 bstring = string.encode(charset) 213 if encoding is None: 214 qlen = _cte_encode_length['q'](bstring) 215 blen = _cte_encode_length['b'](bstring) 216 # Bias toward q. 5 is arbitrary. 217 encoding = 'q' if qlen - blen < 5 else 'b' 218 encoded = _cte_encoders[encoding](bstring) 219 if lang: 220 lang = '*' + lang 221 return "=?{}{}?{}?{}?=".format(charset, lang, encoding, encoded) 222