1# Copyright (C) 2001-2010 Python Software Foundation 2# Author: Barry Warsaw 3# Contact: email-sig@python.org 4 5"""Classes to generate plain text from a message object tree.""" 6 7__all__ = ['Generator', 'DecodedGenerator', 'BytesGenerator'] 8 9import re 10import sys 11import time 12import random 13 14from copy import deepcopy 15from io import StringIO, BytesIO 16from email.utils import _has_surrogates 17 18UNDERSCORE = '_' 19NL = '\n' # XXX: no longer used by the code below. 20 21NLCRE = re.compile(r'\r\n|\r|\n') 22fcre = re.compile(r'^From ', re.MULTILINE) 23 24 25 26class Generator: 27 """Generates output from a Message object tree. 28 29 This basic generator writes the message to the given file object as plain 30 text. 31 """ 32 # 33 # Public interface 34 # 35 36 def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, 37 policy=None): 38 """Create the generator for message flattening. 39 40 outfp is the output file-like object for writing the message to. It 41 must have a write() method. 42 43 Optional mangle_from_ is a flag that, when True (the default if policy 44 is not set), escapes From_ lines in the body of the message by putting 45 a `>' in front of them. 46 47 Optional maxheaderlen specifies the longest length for a non-continued 48 header. When a header line is longer (in characters, with tabs 49 expanded to 8 spaces) than maxheaderlen, the header will split as 50 defined in the Header class. Set maxheaderlen to zero to disable 51 header wrapping. The default is 78, as recommended (but not required) 52 by RFC 2822. 53 54 The policy keyword specifies a policy object that controls a number of 55 aspects of the generator's operation. If no policy is specified, 56 the policy associated with the Message object passed to the 57 flatten method is used. 58 59 """ 60 61 if mangle_from_ is None: 62 mangle_from_ = True if policy is None else policy.mangle_from_ 63 self._fp = outfp 64 self._mangle_from_ = mangle_from_ 65 self.maxheaderlen = maxheaderlen 66 self.policy = policy 67 68 def write(self, s): 69 # Just delegate to the file object 70 self._fp.write(s) 71 72 def flatten(self, msg, unixfrom=False, linesep=None): 73 r"""Print the message object tree rooted at msg to the output file 74 specified when the Generator instance was created. 75 76 unixfrom is a flag that forces the printing of a Unix From_ delimiter 77 before the first object in the message tree. If the original message 78 has no From_ delimiter, a `standard' one is crafted. By default, this 79 is False to inhibit the printing of any From_ delimiter. 80 81 Note that for subobjects, no From_ line is printed. 82 83 linesep specifies the characters used to indicate a new line in 84 the output. The default value is determined by the policy specified 85 when the Generator instance was created or, if none was specified, 86 from the policy associated with the msg. 87 88 """ 89 # We use the _XXX constants for operating on data that comes directly 90 # from the msg, and _encoded_XXX constants for operating on data that 91 # has already been converted (to bytes in the BytesGenerator) and 92 # inserted into a temporary buffer. 93 policy = msg.policy if self.policy is None else self.policy 94 if linesep is not None: 95 policy = policy.clone(linesep=linesep) 96 if self.maxheaderlen is not None: 97 policy = policy.clone(max_line_length=self.maxheaderlen) 98 self._NL = policy.linesep 99 self._encoded_NL = self._encode(self._NL) 100 self._EMPTY = '' 101 self._encoded_EMPTY = self._encode(self._EMPTY) 102 # Because we use clone (below) when we recursively process message 103 # subparts, and because clone uses the computed policy (not None), 104 # submessages will automatically get set to the computed policy when 105 # they are processed by this code. 106 old_gen_policy = self.policy 107 old_msg_policy = msg.policy 108 try: 109 self.policy = policy 110 msg.policy = policy 111 if unixfrom: 112 ufrom = msg.get_unixfrom() 113 if not ufrom: 114 ufrom = 'From nobody ' + time.ctime(time.time()) 115 self.write(ufrom + self._NL) 116 self._write(msg) 117 finally: 118 self.policy = old_gen_policy 119 msg.policy = old_msg_policy 120 121 def clone(self, fp): 122 """Clone this generator with the exact same options.""" 123 return self.__class__(fp, 124 self._mangle_from_, 125 None, # Use policy setting, which we've adjusted 126 policy=self.policy) 127 128 # 129 # Protected interface - undocumented ;/ 130 # 131 132 # Note that we use 'self.write' when what we are writing is coming from 133 # the source, and self._fp.write when what we are writing is coming from a 134 # buffer (because the Bytes subclass has already had a chance to transform 135 # the data in its write method in that case). This is an entirely 136 # pragmatic split determined by experiment; we could be more general by 137 # always using write and having the Bytes subclass write method detect when 138 # it has already transformed the input; but, since this whole thing is a 139 # hack anyway this seems good enough. 140 141 def _new_buffer(self): 142 # BytesGenerator overrides this to return BytesIO. 143 return StringIO() 144 145 def _encode(self, s): 146 # BytesGenerator overrides this to encode strings to bytes. 147 return s 148 149 def _write_lines(self, lines): 150 # We have to transform the line endings. 151 if not lines: 152 return 153 lines = NLCRE.split(lines) 154 for line in lines[:-1]: 155 self.write(line) 156 self.write(self._NL) 157 if lines[-1]: 158 self.write(lines[-1]) 159 # XXX logic tells me this else should be needed, but the tests fail 160 # with it and pass without it. (NLCRE.split ends with a blank element 161 # if and only if there was a trailing newline.) 162 #else: 163 # self.write(self._NL) 164 165 def _write(self, msg): 166 # We can't write the headers yet because of the following scenario: 167 # say a multipart message includes the boundary string somewhere in 168 # its body. We'd have to calculate the new boundary /before/ we write 169 # the headers so that we can write the correct Content-Type: 170 # parameter. 171 # 172 # The way we do this, so as to make the _handle_*() methods simpler, 173 # is to cache any subpart writes into a buffer. The we write the 174 # headers and the buffer contents. That way, subpart handlers can 175 # Do The Right Thing, and can still modify the Content-Type: header if 176 # necessary. 177 oldfp = self._fp 178 try: 179 self._munge_cte = None 180 self._fp = sfp = self._new_buffer() 181 self._dispatch(msg) 182 finally: 183 self._fp = oldfp 184 munge_cte = self._munge_cte 185 del self._munge_cte 186 # If we munged the cte, copy the message again and re-fix the CTE. 187 if munge_cte: 188 msg = deepcopy(msg) 189 # Preserve the header order if the CTE header already exists. 190 if msg.get('content-transfer-encoding') is None: 191 msg['Content-Transfer-Encoding'] = munge_cte[0] 192 else: 193 msg.replace_header('content-transfer-encoding', munge_cte[0]) 194 msg.replace_header('content-type', munge_cte[1]) 195 # Write the headers. First we see if the message object wants to 196 # handle that itself. If not, we'll do it generically. 197 meth = getattr(msg, '_write_headers', None) 198 if meth is None: 199 self._write_headers(msg) 200 else: 201 meth(self) 202 self._fp.write(sfp.getvalue()) 203 204 def _dispatch(self, msg): 205 # Get the Content-Type: for the message, then try to dispatch to 206 # self._handle_<maintype>_<subtype>(). If there's no handler for the 207 # full MIME type, then dispatch to self._handle_<maintype>(). If 208 # that's missing too, then dispatch to self._writeBody(). 209 main = msg.get_content_maintype() 210 sub = msg.get_content_subtype() 211 specific = UNDERSCORE.join((main, sub)).replace('-', '_') 212 meth = getattr(self, '_handle_' + specific, None) 213 if meth is None: 214 generic = main.replace('-', '_') 215 meth = getattr(self, '_handle_' + generic, None) 216 if meth is None: 217 meth = self._writeBody 218 meth(msg) 219 220 # 221 # Default handlers 222 # 223 224 def _write_headers(self, msg): 225 for h, v in msg.raw_items(): 226 self.write(self.policy.fold(h, v)) 227 # A blank line always separates headers from body 228 self.write(self._NL) 229 230 # 231 # Handlers for writing types and subtypes 232 # 233 234 def _handle_text(self, msg): 235 payload = msg.get_payload() 236 if payload is None: 237 return 238 if not isinstance(payload, str): 239 raise TypeError('string payload expected: %s' % type(payload)) 240 if _has_surrogates(msg._payload): 241 charset = msg.get_param('charset') 242 if charset is not None: 243 # XXX: This copy stuff is an ugly hack to avoid modifying the 244 # existing message. 245 msg = deepcopy(msg) 246 del msg['content-transfer-encoding'] 247 msg.set_payload(payload, charset) 248 payload = msg.get_payload() 249 self._munge_cte = (msg['content-transfer-encoding'], 250 msg['content-type']) 251 if self._mangle_from_: 252 payload = fcre.sub('>From ', payload) 253 self._write_lines(payload) 254 255 # Default body handler 256 _writeBody = _handle_text 257 258 def _handle_multipart(self, msg): 259 # The trick here is to write out each part separately, merge them all 260 # together, and then make sure that the boundary we've chosen isn't 261 # present in the payload. 262 msgtexts = [] 263 subparts = msg.get_payload() 264 if subparts is None: 265 subparts = [] 266 elif isinstance(subparts, str): 267 # e.g. a non-strict parse of a message with no starting boundary. 268 self.write(subparts) 269 return 270 elif not isinstance(subparts, list): 271 # Scalar payload 272 subparts = [subparts] 273 for part in subparts: 274 s = self._new_buffer() 275 g = self.clone(s) 276 g.flatten(part, unixfrom=False, linesep=self._NL) 277 msgtexts.append(s.getvalue()) 278 # BAW: What about boundaries that are wrapped in double-quotes? 279 boundary = msg.get_boundary() 280 if not boundary: 281 # Create a boundary that doesn't appear in any of the 282 # message texts. 283 alltext = self._encoded_NL.join(msgtexts) 284 boundary = self._make_boundary(alltext) 285 msg.set_boundary(boundary) 286 # If there's a preamble, write it out, with a trailing CRLF 287 if msg.preamble is not None: 288 if self._mangle_from_: 289 preamble = fcre.sub('>From ', msg.preamble) 290 else: 291 preamble = msg.preamble 292 self._write_lines(preamble) 293 self.write(self._NL) 294 # dash-boundary transport-padding CRLF 295 self.write('--' + boundary + self._NL) 296 # body-part 297 if msgtexts: 298 self._fp.write(msgtexts.pop(0)) 299 # *encapsulation 300 # --> delimiter transport-padding 301 # --> CRLF body-part 302 for body_part in msgtexts: 303 # delimiter transport-padding CRLF 304 self.write(self._NL + '--' + boundary + self._NL) 305 # body-part 306 self._fp.write(body_part) 307 # close-delimiter transport-padding 308 self.write(self._NL + '--' + boundary + '--' + self._NL) 309 if msg.epilogue is not None: 310 if self._mangle_from_: 311 epilogue = fcre.sub('>From ', msg.epilogue) 312 else: 313 epilogue = msg.epilogue 314 self._write_lines(epilogue) 315 316 def _handle_multipart_signed(self, msg): 317 # The contents of signed parts has to stay unmodified in order to keep 318 # the signature intact per RFC1847 2.1, so we disable header wrapping. 319 # RDM: This isn't enough to completely preserve the part, but it helps. 320 p = self.policy 321 self.policy = p.clone(max_line_length=0) 322 try: 323 self._handle_multipart(msg) 324 finally: 325 self.policy = p 326 327 def _handle_message_delivery_status(self, msg): 328 # We can't just write the headers directly to self's file object 329 # because this will leave an extra newline between the last header 330 # block and the boundary. Sigh. 331 blocks = [] 332 for part in msg.get_payload(): 333 s = self._new_buffer() 334 g = self.clone(s) 335 g.flatten(part, unixfrom=False, linesep=self._NL) 336 text = s.getvalue() 337 lines = text.split(self._encoded_NL) 338 # Strip off the unnecessary trailing empty line 339 if lines and lines[-1] == self._encoded_EMPTY: 340 blocks.append(self._encoded_NL.join(lines[:-1])) 341 else: 342 blocks.append(text) 343 # Now join all the blocks with an empty line. This has the lovely 344 # effect of separating each block with an empty line, but not adding 345 # an extra one after the last one. 346 self._fp.write(self._encoded_NL.join(blocks)) 347 348 def _handle_message(self, msg): 349 s = self._new_buffer() 350 g = self.clone(s) 351 # The payload of a message/rfc822 part should be a multipart sequence 352 # of length 1. The zeroth element of the list should be the Message 353 # object for the subpart. Extract that object, stringify it, and 354 # write it out. 355 # Except, it turns out, when it's a string instead, which happens when 356 # and only when HeaderParser is used on a message of mime type 357 # message/rfc822. Such messages are generated by, for example, 358 # Groupwise when forwarding unadorned messages. (Issue 7970.) So 359 # in that case we just emit the string body. 360 payload = msg._payload 361 if isinstance(payload, list): 362 g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL) 363 payload = s.getvalue() 364 else: 365 payload = self._encode(payload) 366 self._fp.write(payload) 367 368 # This used to be a module level function; we use a classmethod for this 369 # and _compile_re so we can continue to provide the module level function 370 # for backward compatibility by doing 371 # _make_boundary = Generator._make_boundary 372 # at the end of the module. It *is* internal, so we could drop that... 373 @classmethod 374 def _make_boundary(cls, text=None): 375 # Craft a random boundary. If text is given, ensure that the chosen 376 # boundary doesn't appear in the text. 377 token = random.randrange(sys.maxsize) 378 boundary = ('=' * 15) + (_fmt % token) + '==' 379 if text is None: 380 return boundary 381 b = boundary 382 counter = 0 383 while True: 384 cre = cls._compile_re('^--' + re.escape(b) + '(--)?$', re.MULTILINE) 385 if not cre.search(text): 386 break 387 b = boundary + '.' + str(counter) 388 counter += 1 389 return b 390 391 @classmethod 392 def _compile_re(cls, s, flags): 393 return re.compile(s, flags) 394 395 396class BytesGenerator(Generator): 397 """Generates a bytes version of a Message object tree. 398 399 Functionally identical to the base Generator except that the output is 400 bytes and not string. When surrogates were used in the input to encode 401 bytes, these are decoded back to bytes for output. If the policy has 402 cte_type set to 7bit, then the message is transformed such that the 403 non-ASCII bytes are properly content transfer encoded, using the charset 404 unknown-8bit. 405 406 The outfp object must accept bytes in its write method. 407 """ 408 409 def write(self, s): 410 self._fp.write(s.encode('ascii', 'surrogateescape')) 411 412 def _new_buffer(self): 413 return BytesIO() 414 415 def _encode(self, s): 416 return s.encode('ascii') 417 418 def _write_headers(self, msg): 419 # This is almost the same as the string version, except for handling 420 # strings with 8bit bytes. 421 for h, v in msg.raw_items(): 422 self._fp.write(self.policy.fold_binary(h, v)) 423 # A blank line always separates headers from body 424 self.write(self._NL) 425 426 def _handle_text(self, msg): 427 # If the string has surrogates the original source was bytes, so 428 # just write it back out. 429 if msg._payload is None: 430 return 431 if _has_surrogates(msg._payload) and not self.policy.cte_type=='7bit': 432 if self._mangle_from_: 433 msg._payload = fcre.sub(">From ", msg._payload) 434 self._write_lines(msg._payload) 435 else: 436 super(BytesGenerator,self)._handle_text(msg) 437 438 # Default body handler 439 _writeBody = _handle_text 440 441 @classmethod 442 def _compile_re(cls, s, flags): 443 return re.compile(s.encode('ascii'), flags) 444 445 446 447_FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' 448 449class DecodedGenerator(Generator): 450 """Generates a text representation of a message. 451 452 Like the Generator base class, except that non-text parts are substituted 453 with a format string representing the part. 454 """ 455 def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, fmt=None, *, 456 policy=None): 457 """Like Generator.__init__() except that an additional optional 458 argument is allowed. 459 460 Walks through all subparts of a message. If the subpart is of main 461 type `text', then it prints the decoded payload of the subpart. 462 463 Otherwise, fmt is a format string that is used instead of the message 464 payload. fmt is expanded with the following keywords (in 465 %(keyword)s format): 466 467 type : Full MIME type of the non-text part 468 maintype : Main MIME type of the non-text part 469 subtype : Sub-MIME type of the non-text part 470 filename : Filename of the non-text part 471 description: Description associated with the non-text part 472 encoding : Content transfer encoding of the non-text part 473 474 The default value for fmt is None, meaning 475 476 [Non-text (%(type)s) part of message omitted, filename %(filename)s] 477 """ 478 Generator.__init__(self, outfp, mangle_from_, maxheaderlen, 479 policy=policy) 480 if fmt is None: 481 self._fmt = _FMT 482 else: 483 self._fmt = fmt 484 485 def _dispatch(self, msg): 486 for part in msg.walk(): 487 maintype = part.get_content_maintype() 488 if maintype == 'text': 489 print(part.get_payload(decode=False), file=self) 490 elif maintype == 'multipart': 491 # Just skip this 492 pass 493 else: 494 print(self._fmt % { 495 'type' : part.get_content_type(), 496 'maintype' : part.get_content_maintype(), 497 'subtype' : part.get_content_subtype(), 498 'filename' : part.get_filename('[no filename]'), 499 'description': part.get('Content-Description', 500 '[no description]'), 501 'encoding' : part.get('Content-Transfer-Encoding', 502 '[no encoding]'), 503 }, file=self) 504 505 506 507# Helper used by Generator._make_boundary 508_width = len(repr(sys.maxsize-1)) 509_fmt = '%%0%dd' % _width 510 511# Backward compatibility 512_make_boundary = Generator._make_boundary 513