1# Copyright (C) 2001-2010 Python Software Foundation 2# Author: Barry Warsaw 3# Contact: email-sig@python.org 4 5"""Classes to generate plain text from a message object tree.""" 6 7__all__ = ['Generator', 'DecodedGenerator', 'BytesGenerator'] 8 9import re 10import sys 11import time 12import random 13 14from copy import deepcopy 15from io import StringIO, BytesIO 16from email.utils import _has_surrogates 17from email.errors import HeaderWriteError 18 19UNDERSCORE = '_' 20NL = '\n' # XXX: no longer used by the code below. 21 22NLCRE = re.compile(r'\r\n|\r|\n') 23fcre = re.compile(r'^From ', re.MULTILINE) 24NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') 25 26 27class Generator: 28 """Generates output from a Message object tree. 29 30 This basic generator writes the message to the given file object as plain 31 text. 32 """ 33 # 34 # Public interface 35 # 36 37 def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, 38 policy=None): 39 """Create the generator for message flattening. 40 41 outfp is the output file-like object for writing the message to. It 42 must have a write() method. 43 44 Optional mangle_from_ is a flag that, when True (the default if policy 45 is not set), escapes From_ lines in the body of the message by putting 46 a `>' in front of them. 47 48 Optional maxheaderlen specifies the longest length for a non-continued 49 header. When a header line is longer (in characters, with tabs 50 expanded to 8 spaces) than maxheaderlen, the header will split as 51 defined in the Header class. Set maxheaderlen to zero to disable 52 header wrapping. The default is 78, as recommended (but not required) 53 by RFC 2822. 54 55 The policy keyword specifies a policy object that controls a number of 56 aspects of the generator's operation. If no policy is specified, 57 the policy associated with the Message object passed to the 58 flatten method is used. 59 60 """ 61 62 if mangle_from_ is None: 63 mangle_from_ = True if policy is None else policy.mangle_from_ 64 self._fp = outfp 65 self._mangle_from_ = mangle_from_ 66 self.maxheaderlen = maxheaderlen 67 self.policy = policy 68 69 def write(self, s): 70 # Just delegate to the file object 71 self._fp.write(s) 72 73 def flatten(self, msg, unixfrom=False, linesep=None): 74 r"""Print the message object tree rooted at msg to the output file 75 specified when the Generator instance was created. 76 77 unixfrom is a flag that forces the printing of a Unix From_ delimiter 78 before the first object in the message tree. If the original message 79 has no From_ delimiter, a `standard' one is crafted. By default, this 80 is False to inhibit the printing of any From_ delimiter. 81 82 Note that for subobjects, no From_ line is printed. 83 84 linesep specifies the characters used to indicate a new line in 85 the output. The default value is determined by the policy specified 86 when the Generator instance was created or, if none was specified, 87 from the policy associated with the msg. 88 89 """ 90 # We use the _XXX constants for operating on data that comes directly 91 # from the msg, and _encoded_XXX constants for operating on data that 92 # has already been converted (to bytes in the BytesGenerator) and 93 # inserted into a temporary buffer. 94 policy = msg.policy if self.policy is None else self.policy 95 if linesep is not None: 96 policy = policy.clone(linesep=linesep) 97 if self.maxheaderlen is not None: 98 policy = policy.clone(max_line_length=self.maxheaderlen) 99 self._NL = policy.linesep 100 self._encoded_NL = self._encode(self._NL) 101 self._EMPTY = '' 102 self._encoded_EMPTY = self._encode(self._EMPTY) 103 # Because we use clone (below) when we recursively process message 104 # subparts, and because clone uses the computed policy (not None), 105 # submessages will automatically get set to the computed policy when 106 # they are processed by this code. 107 old_gen_policy = self.policy 108 old_msg_policy = msg.policy 109 try: 110 self.policy = policy 111 msg.policy = policy 112 if unixfrom: 113 ufrom = msg.get_unixfrom() 114 if not ufrom: 115 ufrom = 'From nobody ' + time.ctime(time.time()) 116 self.write(ufrom + self._NL) 117 self._write(msg) 118 finally: 119 self.policy = old_gen_policy 120 msg.policy = old_msg_policy 121 122 def clone(self, fp): 123 """Clone this generator with the exact same options.""" 124 return self.__class__(fp, 125 self._mangle_from_, 126 None, # Use policy setting, which we've adjusted 127 policy=self.policy) 128 129 # 130 # Protected interface - undocumented ;/ 131 # 132 133 # Note that we use 'self.write' when what we are writing is coming from 134 # the source, and self._fp.write when what we are writing is coming from a 135 # buffer (because the Bytes subclass has already had a chance to transform 136 # the data in its write method in that case). This is an entirely 137 # pragmatic split determined by experiment; we could be more general by 138 # always using write and having the Bytes subclass write method detect when 139 # it has already transformed the input; but, since this whole thing is a 140 # hack anyway this seems good enough. 141 142 def _new_buffer(self): 143 # BytesGenerator overrides this to return BytesIO. 144 return StringIO() 145 146 def _encode(self, s): 147 # BytesGenerator overrides this to encode strings to bytes. 148 return s 149 150 def _write_lines(self, lines): 151 # We have to transform the line endings. 152 if not lines: 153 return 154 lines = NLCRE.split(lines) 155 for line in lines[:-1]: 156 self.write(line) 157 self.write(self._NL) 158 if lines[-1]: 159 self.write(lines[-1]) 160 # XXX logic tells me this else should be needed, but the tests fail 161 # with it and pass without it. (NLCRE.split ends with a blank element 162 # if and only if there was a trailing newline.) 163 #else: 164 # self.write(self._NL) 165 166 def _write(self, msg): 167 # We can't write the headers yet because of the following scenario: 168 # say a multipart message includes the boundary string somewhere in 169 # its body. We'd have to calculate the new boundary /before/ we write 170 # the headers so that we can write the correct Content-Type: 171 # parameter. 172 # 173 # The way we do this, so as to make the _handle_*() methods simpler, 174 # is to cache any subpart writes into a buffer. Then we write the 175 # headers and the buffer contents. That way, subpart handlers can 176 # Do The Right Thing, and can still modify the Content-Type: header if 177 # necessary. 178 oldfp = self._fp 179 try: 180 self._munge_cte = None 181 self._fp = sfp = self._new_buffer() 182 self._dispatch(msg) 183 finally: 184 self._fp = oldfp 185 munge_cte = self._munge_cte 186 del self._munge_cte 187 # If we munged the cte, copy the message again and re-fix the CTE. 188 if munge_cte: 189 msg = deepcopy(msg) 190 # Preserve the header order if the CTE header already exists. 191 if msg.get('content-transfer-encoding') is None: 192 msg['Content-Transfer-Encoding'] = munge_cte[0] 193 else: 194 msg.replace_header('content-transfer-encoding', munge_cte[0]) 195 msg.replace_header('content-type', munge_cte[1]) 196 # Write the headers. First we see if the message object wants to 197 # handle that itself. If not, we'll do it generically. 198 meth = getattr(msg, '_write_headers', None) 199 if meth is None: 200 self._write_headers(msg) 201 else: 202 meth(self) 203 self._fp.write(sfp.getvalue()) 204 205 def _dispatch(self, msg): 206 # Get the Content-Type: for the message, then try to dispatch to 207 # self._handle_<maintype>_<subtype>(). If there's no handler for the 208 # full MIME type, then dispatch to self._handle_<maintype>(). If 209 # that's missing too, then dispatch to self._writeBody(). 210 main = msg.get_content_maintype() 211 sub = msg.get_content_subtype() 212 specific = UNDERSCORE.join((main, sub)).replace('-', '_') 213 meth = getattr(self, '_handle_' + specific, None) 214 if meth is None: 215 generic = main.replace('-', '_') 216 meth = getattr(self, '_handle_' + generic, None) 217 if meth is None: 218 meth = self._writeBody 219 meth(msg) 220 221 # 222 # Default handlers 223 # 224 225 def _write_headers(self, msg): 226 for h, v in msg.raw_items(): 227 folded = self.policy.fold(h, v) 228 if self.policy.verify_generated_headers: 229 linesep = self.policy.linesep 230 if not folded.endswith(self.policy.linesep): 231 raise HeaderWriteError( 232 f'folded header does not end with {linesep!r}: {folded!r}') 233 if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)): 234 raise HeaderWriteError( 235 f'folded header contains newline: {folded!r}') 236 self.write(folded) 237 # A blank line always separates headers from body 238 self.write(self._NL) 239 240 # 241 # Handlers for writing types and subtypes 242 # 243 244 def _handle_text(self, msg): 245 payload = msg.get_payload() 246 if payload is None: 247 return 248 if not isinstance(payload, str): 249 raise TypeError('string payload expected: %s' % type(payload)) 250 if _has_surrogates(msg._payload): 251 charset = msg.get_param('charset') 252 if charset is not None: 253 # XXX: This copy stuff is an ugly hack to avoid modifying the 254 # existing message. 255 msg = deepcopy(msg) 256 del msg['content-transfer-encoding'] 257 msg.set_payload(msg._payload, charset) 258 payload = msg.get_payload() 259 self._munge_cte = (msg['content-transfer-encoding'], 260 msg['content-type']) 261 if self._mangle_from_: 262 payload = fcre.sub('>From ', payload) 263 self._write_lines(payload) 264 265 # Default body handler 266 _writeBody = _handle_text 267 268 def _handle_multipart(self, msg): 269 # The trick here is to write out each part separately, merge them all 270 # together, and then make sure that the boundary we've chosen isn't 271 # present in the payload. 272 msgtexts = [] 273 subparts = msg.get_payload() 274 if subparts is None: 275 subparts = [] 276 elif isinstance(subparts, str): 277 # e.g. a non-strict parse of a message with no starting boundary. 278 self.write(subparts) 279 return 280 elif not isinstance(subparts, list): 281 # Scalar payload 282 subparts = [subparts] 283 for part in subparts: 284 s = self._new_buffer() 285 g = self.clone(s) 286 g.flatten(part, unixfrom=False, linesep=self._NL) 287 msgtexts.append(s.getvalue()) 288 # BAW: What about boundaries that are wrapped in double-quotes? 289 boundary = msg.get_boundary() 290 if not boundary: 291 # Create a boundary that doesn't appear in any of the 292 # message texts. 293 alltext = self._encoded_NL.join(msgtexts) 294 boundary = self._make_boundary(alltext) 295 msg.set_boundary(boundary) 296 # If there's a preamble, write it out, with a trailing CRLF 297 if msg.preamble is not None: 298 if self._mangle_from_: 299 preamble = fcre.sub('>From ', msg.preamble) 300 else: 301 preamble = msg.preamble 302 self._write_lines(preamble) 303 self.write(self._NL) 304 # dash-boundary transport-padding CRLF 305 self.write('--' + boundary + self._NL) 306 # body-part 307 if msgtexts: 308 self._fp.write(msgtexts.pop(0)) 309 # *encapsulation 310 # --> delimiter transport-padding 311 # --> CRLF body-part 312 for body_part in msgtexts: 313 # delimiter transport-padding CRLF 314 self.write(self._NL + '--' + boundary + self._NL) 315 # body-part 316 self._fp.write(body_part) 317 # close-delimiter transport-padding 318 self.write(self._NL + '--' + boundary + '--' + self._NL) 319 if msg.epilogue is not None: 320 if self._mangle_from_: 321 epilogue = fcre.sub('>From ', msg.epilogue) 322 else: 323 epilogue = msg.epilogue 324 self._write_lines(epilogue) 325 326 def _handle_multipart_signed(self, msg): 327 # The contents of signed parts has to stay unmodified in order to keep 328 # the signature intact per RFC1847 2.1, so we disable header wrapping. 329 # RDM: This isn't enough to completely preserve the part, but it helps. 330 p = self.policy 331 self.policy = p.clone(max_line_length=0) 332 try: 333 self._handle_multipart(msg) 334 finally: 335 self.policy = p 336 337 def _handle_message_delivery_status(self, msg): 338 # We can't just write the headers directly to self's file object 339 # because this will leave an extra newline between the last header 340 # block and the boundary. Sigh. 341 blocks = [] 342 for part in msg.get_payload(): 343 s = self._new_buffer() 344 g = self.clone(s) 345 g.flatten(part, unixfrom=False, linesep=self._NL) 346 text = s.getvalue() 347 lines = text.split(self._encoded_NL) 348 # Strip off the unnecessary trailing empty line 349 if lines and lines[-1] == self._encoded_EMPTY: 350 blocks.append(self._encoded_NL.join(lines[:-1])) 351 else: 352 blocks.append(text) 353 # Now join all the blocks with an empty line. This has the lovely 354 # effect of separating each block with an empty line, but not adding 355 # an extra one after the last one. 356 self._fp.write(self._encoded_NL.join(blocks)) 357 358 def _handle_message(self, msg): 359 s = self._new_buffer() 360 g = self.clone(s) 361 # The payload of a message/rfc822 part should be a multipart sequence 362 # of length 1. The zeroth element of the list should be the Message 363 # object for the subpart. Extract that object, stringify it, and 364 # write it out. 365 # Except, it turns out, when it's a string instead, which happens when 366 # and only when HeaderParser is used on a message of mime type 367 # message/rfc822. Such messages are generated by, for example, 368 # Groupwise when forwarding unadorned messages. (Issue 7970.) So 369 # in that case we just emit the string body. 370 payload = msg._payload 371 if isinstance(payload, list): 372 g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL) 373 payload = s.getvalue() 374 else: 375 payload = self._encode(payload) 376 self._fp.write(payload) 377 378 # This used to be a module level function; we use a classmethod for this 379 # and _compile_re so we can continue to provide the module level function 380 # for backward compatibility by doing 381 # _make_boundary = Generator._make_boundary 382 # at the end of the module. It *is* internal, so we could drop that... 383 @classmethod 384 def _make_boundary(cls, text=None): 385 # Craft a random boundary. If text is given, ensure that the chosen 386 # boundary doesn't appear in the text. 387 token = random.randrange(sys.maxsize) 388 boundary = ('=' * 15) + (_fmt % token) + '==' 389 if text is None: 390 return boundary 391 b = boundary 392 counter = 0 393 while True: 394 cre = cls._compile_re('^--' + re.escape(b) + '(--)?$', re.MULTILINE) 395 if not cre.search(text): 396 break 397 b = boundary + '.' + str(counter) 398 counter += 1 399 return b 400 401 @classmethod 402 def _compile_re(cls, s, flags): 403 return re.compile(s, flags) 404 405 406class BytesGenerator(Generator): 407 """Generates a bytes version of a Message object tree. 408 409 Functionally identical to the base Generator except that the output is 410 bytes and not string. When surrogates were used in the input to encode 411 bytes, these are decoded back to bytes for output. If the policy has 412 cte_type set to 7bit, then the message is transformed such that the 413 non-ASCII bytes are properly content transfer encoded, using the charset 414 unknown-8bit. 415 416 The outfp object must accept bytes in its write method. 417 """ 418 419 def write(self, s): 420 self._fp.write(s.encode('ascii', 'surrogateescape')) 421 422 def _new_buffer(self): 423 return BytesIO() 424 425 def _encode(self, s): 426 return s.encode('ascii') 427 428 def _write_headers(self, msg): 429 # This is almost the same as the string version, except for handling 430 # strings with 8bit bytes. 431 for h, v in msg.raw_items(): 432 self._fp.write(self.policy.fold_binary(h, v)) 433 # A blank line always separates headers from body 434 self.write(self._NL) 435 436 def _handle_text(self, msg): 437 # If the string has surrogates the original source was bytes, so 438 # just write it back out. 439 if msg._payload is None: 440 return 441 if _has_surrogates(msg._payload) and not self.policy.cte_type=='7bit': 442 if self._mangle_from_: 443 msg._payload = fcre.sub(">From ", msg._payload) 444 self._write_lines(msg._payload) 445 else: 446 super(BytesGenerator,self)._handle_text(msg) 447 448 # Default body handler 449 _writeBody = _handle_text 450 451 @classmethod 452 def _compile_re(cls, s, flags): 453 return re.compile(s.encode('ascii'), flags) 454 455 456_FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' 457 458class DecodedGenerator(Generator): 459 """Generates a text representation of a message. 460 461 Like the Generator base class, except that non-text parts are substituted 462 with a format string representing the part. 463 """ 464 def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, fmt=None, *, 465 policy=None): 466 """Like Generator.__init__() except that an additional optional 467 argument is allowed. 468 469 Walks through all subparts of a message. If the subpart is of main 470 type `text', then it prints the decoded payload of the subpart. 471 472 Otherwise, fmt is a format string that is used instead of the message 473 payload. fmt is expanded with the following keywords (in 474 %(keyword)s format): 475 476 type : Full MIME type of the non-text part 477 maintype : Main MIME type of the non-text part 478 subtype : Sub-MIME type of the non-text part 479 filename : Filename of the non-text part 480 description: Description associated with the non-text part 481 encoding : Content transfer encoding of the non-text part 482 483 The default value for fmt is None, meaning 484 485 [Non-text (%(type)s) part of message omitted, filename %(filename)s] 486 """ 487 Generator.__init__(self, outfp, mangle_from_, maxheaderlen, 488 policy=policy) 489 if fmt is None: 490 self._fmt = _FMT 491 else: 492 self._fmt = fmt 493 494 def _dispatch(self, msg): 495 for part in msg.walk(): 496 maintype = part.get_content_maintype() 497 if maintype == 'text': 498 print(part.get_payload(decode=False), file=self) 499 elif maintype == 'multipart': 500 # Just skip this 501 pass 502 else: 503 print(self._fmt % { 504 'type' : part.get_content_type(), 505 'maintype' : part.get_content_maintype(), 506 'subtype' : part.get_content_subtype(), 507 'filename' : part.get_filename('[no filename]'), 508 'description': part.get('Content-Description', 509 '[no description]'), 510 'encoding' : part.get('Content-Transfer-Encoding', 511 '[no encoding]'), 512 }, file=self) 513 514 515# Helper used by Generator._make_boundary 516_width = len(repr(sys.maxsize-1)) 517_fmt = '%%0%dd' % _width 518 519# Backward compatibility 520_make_boundary = Generator._make_boundary 521