1"""An NNTP client class based on: 2- RFC 977: Network News Transfer Protocol 3- RFC 2980: Common NNTP Extensions 4- RFC 3977: Network News Transfer Protocol (version 2) 5 6Example: 7 8>>> from nntplib import NNTP 9>>> s = NNTP('news') 10>>> resp, count, first, last, name = s.group('comp.lang.python') 11>>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) 12Group comp.lang.python has 51 articles, range 5770 to 5821 13>>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last)) 14>>> resp = s.quit() 15>>> 16 17Here 'resp' is the server response line. 18Error responses are turned into exceptions. 19 20To post an article from a file: 21>>> f = open(filename, 'rb') # file containing article, including header 22>>> resp = s.post(f) 23>>> 24 25For descriptions of all methods, read the comments in the code below. 26Note that all arguments and return values representing article numbers 27are strings, not numbers, since they are rarely used for calculations. 28""" 29 30# RFC 977 by Brian Kantor and Phil Lapsley. 31# xover, xgtitle, xpath, date methods by Kevan Heydon 32 33# Incompatible changes from the 2.x nntplib: 34# - all commands are encoded as UTF-8 data (using the "surrogateescape" 35# error handler), except for raw message data (POST, IHAVE) 36# - all responses are decoded as UTF-8 data (using the "surrogateescape" 37# error handler), except for raw message data (ARTICLE, HEAD, BODY) 38# - the `file` argument to various methods is keyword-only 39# 40# - NNTP.date() returns a datetime object 41# - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object, 42# rather than a pair of (date, time) strings. 43# - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples 44# - NNTP.descriptions() returns a dict mapping group names to descriptions 45# - NNTP.xover() returns a list of dicts mapping field names (header or metadata) 46# to field values; each dict representing a message overview. 47# - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo) 48# tuple. 49# - the "internal" methods have been marked private (they now start with 50# an underscore) 51 52# Other changes from the 2.x/3.1 nntplib: 53# - automatic querying of capabilities at connect 54# - New method NNTP.getcapabilities() 55# - New method NNTP.over() 56# - New helper function decode_header() 57# - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and 58# arbitrary iterables yielding lines. 59# - An extensive test suite :-) 60 61# TODO: 62# - return structured data (GroupInfo etc.) everywhere 63# - support HDR 64 65# Imports 66import re 67import socket 68import collections 69import datetime 70import warnings 71 72try: 73 import ssl 74except ImportError: 75 _have_ssl = False 76else: 77 _have_ssl = True 78 79from email.header import decode_header as _email_decode_header 80from socket import _GLOBAL_DEFAULT_TIMEOUT 81 82__all__ = ["NNTP", 83 "NNTPError", "NNTPReplyError", "NNTPTemporaryError", 84 "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError", 85 "decode_header", 86 ] 87 88# maximal line length when calling readline(). This is to prevent 89# reading arbitrary length lines. RFC 3977 limits NNTP line length to 90# 512 characters, including CRLF. We have selected 2048 just to be on 91# the safe side. 92_MAXLINE = 2048 93 94 95# Exceptions raised when an error or invalid response is received 96class NNTPError(Exception): 97 """Base class for all nntplib exceptions""" 98 def __init__(self, *args): 99 Exception.__init__(self, *args) 100 try: 101 self.response = args[0] 102 except IndexError: 103 self.response = 'No response given' 104 105class NNTPReplyError(NNTPError): 106 """Unexpected [123]xx reply""" 107 pass 108 109class NNTPTemporaryError(NNTPError): 110 """4xx errors""" 111 pass 112 113class NNTPPermanentError(NNTPError): 114 """5xx errors""" 115 pass 116 117class NNTPProtocolError(NNTPError): 118 """Response does not begin with [1-5]""" 119 pass 120 121class NNTPDataError(NNTPError): 122 """Error in response data""" 123 pass 124 125 126# Standard port used by NNTP servers 127NNTP_PORT = 119 128NNTP_SSL_PORT = 563 129 130# Response numbers that are followed by additional text (e.g. article) 131_LONGRESP = { 132 '100', # HELP 133 '101', # CAPABILITIES 134 '211', # LISTGROUP (also not multi-line with GROUP) 135 '215', # LIST 136 '220', # ARTICLE 137 '221', # HEAD, XHDR 138 '222', # BODY 139 '224', # OVER, XOVER 140 '225', # HDR 141 '230', # NEWNEWS 142 '231', # NEWGROUPS 143 '282', # XGTITLE 144} 145 146# Default decoded value for LIST OVERVIEW.FMT if not supported 147_DEFAULT_OVERVIEW_FMT = [ 148 "subject", "from", "date", "message-id", "references", ":bytes", ":lines"] 149 150# Alternative names allowed in LIST OVERVIEW.FMT response 151_OVERVIEW_FMT_ALTERNATIVES = { 152 'bytes': ':bytes', 153 'lines': ':lines', 154} 155 156# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) 157_CRLF = b'\r\n' 158 159GroupInfo = collections.namedtuple('GroupInfo', 160 ['group', 'last', 'first', 'flag']) 161 162ArticleInfo = collections.namedtuple('ArticleInfo', 163 ['number', 'message_id', 'lines']) 164 165 166# Helper function(s) 167def decode_header(header_str): 168 """Takes a unicode string representing a munged header value 169 and decodes it as a (possibly non-ASCII) readable value.""" 170 parts = [] 171 for v, enc in _email_decode_header(header_str): 172 if isinstance(v, bytes): 173 parts.append(v.decode(enc or 'ascii')) 174 else: 175 parts.append(v) 176 return ''.join(parts) 177 178def _parse_overview_fmt(lines): 179 """Parse a list of string representing the response to LIST OVERVIEW.FMT 180 and return a list of header/metadata names. 181 Raises NNTPDataError if the response is not compliant 182 (cf. RFC 3977, section 8.4).""" 183 fmt = [] 184 for line in lines: 185 if line[0] == ':': 186 # Metadata name (e.g. ":bytes") 187 name, _, suffix = line[1:].partition(':') 188 name = ':' + name 189 else: 190 # Header name (e.g. "Subject:" or "Xref:full") 191 name, _, suffix = line.partition(':') 192 name = name.lower() 193 name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name) 194 # Should we do something with the suffix? 195 fmt.append(name) 196 defaults = _DEFAULT_OVERVIEW_FMT 197 if len(fmt) < len(defaults): 198 raise NNTPDataError("LIST OVERVIEW.FMT response too short") 199 if fmt[:len(defaults)] != defaults: 200 raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields") 201 return fmt 202 203def _parse_overview(lines, fmt, data_process_func=None): 204 """Parse the response to an OVER or XOVER command according to the 205 overview format `fmt`.""" 206 n_defaults = len(_DEFAULT_OVERVIEW_FMT) 207 overview = [] 208 for line in lines: 209 fields = {} 210 article_number, *tokens = line.split('\t') 211 article_number = int(article_number) 212 for i, token in enumerate(tokens): 213 if i >= len(fmt): 214 # XXX should we raise an error? Some servers might not 215 # support LIST OVERVIEW.FMT and still return additional 216 # headers. 217 continue 218 field_name = fmt[i] 219 is_metadata = field_name.startswith(':') 220 if i >= n_defaults and not is_metadata: 221 # Non-default header names are included in full in the response 222 # (unless the field is totally empty) 223 h = field_name + ": " 224 if token and token[:len(h)].lower() != h: 225 raise NNTPDataError("OVER/XOVER response doesn't include " 226 "names of additional headers") 227 token = token[len(h):] if token else None 228 fields[fmt[i]] = token 229 overview.append((article_number, fields)) 230 return overview 231 232def _parse_datetime(date_str, time_str=None): 233 """Parse a pair of (date, time) strings, and return a datetime object. 234 If only the date is given, it is assumed to be date and time 235 concatenated together (e.g. response to the DATE command). 236 """ 237 if time_str is None: 238 time_str = date_str[-6:] 239 date_str = date_str[:-6] 240 hours = int(time_str[:2]) 241 minutes = int(time_str[2:4]) 242 seconds = int(time_str[4:]) 243 year = int(date_str[:-4]) 244 month = int(date_str[-4:-2]) 245 day = int(date_str[-2:]) 246 # RFC 3977 doesn't say how to interpret 2-char years. Assume that 247 # there are no dates before 1970 on Usenet. 248 if year < 70: 249 year += 2000 250 elif year < 100: 251 year += 1900 252 return datetime.datetime(year, month, day, hours, minutes, seconds) 253 254def _unparse_datetime(dt, legacy=False): 255 """Format a date or datetime object as a pair of (date, time) strings 256 in the format required by the NEWNEWS and NEWGROUPS commands. If a 257 date object is passed, the time is assumed to be midnight (00h00). 258 259 The returned representation depends on the legacy flag: 260 * if legacy is False (the default): 261 date has the YYYYMMDD format and time the HHMMSS format 262 * if legacy is True: 263 date has the YYMMDD format and time the HHMMSS format. 264 RFC 3977 compliant servers should understand both formats; therefore, 265 legacy is only needed when talking to old servers. 266 """ 267 if not isinstance(dt, datetime.datetime): 268 time_str = "000000" 269 else: 270 time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt) 271 y = dt.year 272 if legacy: 273 y = y % 100 274 date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt) 275 else: 276 date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt) 277 return date_str, time_str 278 279 280if _have_ssl: 281 282 def _encrypt_on(sock, context, hostname): 283 """Wrap a socket in SSL/TLS. Arguments: 284 - sock: Socket to wrap 285 - context: SSL context to use for the encrypted connection 286 Returns: 287 - sock: New, encrypted socket. 288 """ 289 # Generate a default SSL context if none was passed. 290 if context is None: 291 context = ssl._create_stdlib_context() 292 return context.wrap_socket(sock, server_hostname=hostname) 293 294 295# The classes themselves 296class _NNTPBase: 297 # UTF-8 is the character set for all NNTP commands and responses: they 298 # are automatically encoded (when sending) and decoded (and receiving) 299 # by this class. 300 # However, some multi-line data blocks can contain arbitrary bytes (for 301 # example, latin-1 or utf-16 data in the body of a message). Commands 302 # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message 303 # data will therefore only accept and produce bytes objects. 304 # Furthermore, since there could be non-compliant servers out there, 305 # we use 'surrogateescape' as the error handler for fault tolerance 306 # and easy round-tripping. This could be useful for some applications 307 # (e.g. NNTP gateways). 308 309 encoding = 'utf-8' 310 errors = 'surrogateescape' 311 312 def __init__(self, file, host, 313 readermode=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): 314 """Initialize an instance. Arguments: 315 - file: file-like object (open for read/write in binary mode) 316 - host: hostname of the server 317 - readermode: if true, send 'mode reader' command after 318 connecting. 319 - timeout: timeout (in seconds) used for socket connections 320 321 readermode is sometimes necessary if you are connecting to an 322 NNTP server on the local machine and intend to call 323 reader-specific commands, such as `group'. If you get 324 unexpected NNTPPermanentErrors, you might need to set 325 readermode. 326 """ 327 self.host = host 328 self.file = file 329 self.debugging = 0 330 self.welcome = self._getresp() 331 332 # Inquire about capabilities (RFC 3977). 333 self._caps = None 334 self.getcapabilities() 335 336 # 'MODE READER' is sometimes necessary to enable 'reader' mode. 337 # However, the order in which 'MODE READER' and 'AUTHINFO' need to 338 # arrive differs between some NNTP servers. If _setreadermode() fails 339 # with an authorization failed error, it will set this to True; 340 # the login() routine will interpret that as a request to try again 341 # after performing its normal function. 342 # Enable only if we're not already in READER mode anyway. 343 self.readermode_afterauth = False 344 if readermode and 'READER' not in self._caps: 345 self._setreadermode() 346 if not self.readermode_afterauth: 347 # Capabilities might have changed after MODE READER 348 self._caps = None 349 self.getcapabilities() 350 351 # RFC 4642 2.2.2: Both the client and the server MUST know if there is 352 # a TLS session active. A client MUST NOT attempt to start a TLS 353 # session if a TLS session is already active. 354 self.tls_on = False 355 356 # Log in and encryption setup order is left to subclasses. 357 self.authenticated = False 358 359 def __enter__(self): 360 return self 361 362 def __exit__(self, *args): 363 is_connected = lambda: hasattr(self, "file") 364 if is_connected(): 365 try: 366 self.quit() 367 except (OSError, EOFError): 368 pass 369 finally: 370 if is_connected(): 371 self._close() 372 373 def getwelcome(self): 374 """Get the welcome message from the server 375 (this is read and squirreled away by __init__()). 376 If the response code is 200, posting is allowed; 377 if it 201, posting is not allowed.""" 378 379 if self.debugging: print('*welcome*', repr(self.welcome)) 380 return self.welcome 381 382 def getcapabilities(self): 383 """Get the server capabilities, as read by __init__(). 384 If the CAPABILITIES command is not supported, an empty dict is 385 returned.""" 386 if self._caps is None: 387 self.nntp_version = 1 388 self.nntp_implementation = None 389 try: 390 resp, caps = self.capabilities() 391 except (NNTPPermanentError, NNTPTemporaryError): 392 # Server doesn't support capabilities 393 self._caps = {} 394 else: 395 self._caps = caps 396 if 'VERSION' in caps: 397 # The server can advertise several supported versions, 398 # choose the highest. 399 self.nntp_version = max(map(int, caps['VERSION'])) 400 if 'IMPLEMENTATION' in caps: 401 self.nntp_implementation = ' '.join(caps['IMPLEMENTATION']) 402 return self._caps 403 404 def set_debuglevel(self, level): 405 """Set the debugging level. Argument 'level' means: 406 0: no debugging output (default) 407 1: print commands and responses but not body text etc. 408 2: also print raw lines read and sent before stripping CR/LF""" 409 410 self.debugging = level 411 debug = set_debuglevel 412 413 def _putline(self, line): 414 """Internal: send one line to the server, appending CRLF. 415 The `line` must be a bytes-like object.""" 416 line = line + _CRLF 417 if self.debugging > 1: print('*put*', repr(line)) 418 self.file.write(line) 419 self.file.flush() 420 421 def _putcmd(self, line): 422 """Internal: send one command to the server (through _putline()). 423 The `line` must be a unicode string.""" 424 if self.debugging: print('*cmd*', repr(line)) 425 line = line.encode(self.encoding, self.errors) 426 self._putline(line) 427 428 def _getline(self, strip_crlf=True): 429 """Internal: return one line from the server, stripping _CRLF. 430 Raise EOFError if the connection is closed. 431 Returns a bytes object.""" 432 line = self.file.readline(_MAXLINE +1) 433 if len(line) > _MAXLINE: 434 raise NNTPDataError('line too long') 435 if self.debugging > 1: 436 print('*get*', repr(line)) 437 if not line: raise EOFError 438 if strip_crlf: 439 if line[-2:] == _CRLF: 440 line = line[:-2] 441 elif line[-1:] in _CRLF: 442 line = line[:-1] 443 return line 444 445 def _getresp(self): 446 """Internal: get a response from the server. 447 Raise various errors if the response indicates an error. 448 Returns a unicode string.""" 449 resp = self._getline() 450 if self.debugging: print('*resp*', repr(resp)) 451 resp = resp.decode(self.encoding, self.errors) 452 c = resp[:1] 453 if c == '4': 454 raise NNTPTemporaryError(resp) 455 if c == '5': 456 raise NNTPPermanentError(resp) 457 if c not in '123': 458 raise NNTPProtocolError(resp) 459 return resp 460 461 def _getlongresp(self, file=None): 462 """Internal: get a response plus following text from the server. 463 Raise various errors if the response indicates an error. 464 465 Returns a (response, lines) tuple where `response` is a unicode 466 string and `lines` is a list of bytes objects. 467 If `file` is a file-like object, it must be open in binary mode. 468 """ 469 470 openedFile = None 471 try: 472 # If a string was passed then open a file with that name 473 if isinstance(file, (str, bytes)): 474 openedFile = file = open(file, "wb") 475 476 resp = self._getresp() 477 if resp[:3] not in _LONGRESP: 478 raise NNTPReplyError(resp) 479 480 lines = [] 481 if file is not None: 482 # XXX lines = None instead? 483 terminators = (b'.' + _CRLF, b'.\n') 484 while 1: 485 line = self._getline(False) 486 if line in terminators: 487 break 488 if line.startswith(b'..'): 489 line = line[1:] 490 file.write(line) 491 else: 492 terminator = b'.' 493 while 1: 494 line = self._getline() 495 if line == terminator: 496 break 497 if line.startswith(b'..'): 498 line = line[1:] 499 lines.append(line) 500 finally: 501 # If this method created the file, then it must close it 502 if openedFile: 503 openedFile.close() 504 505 return resp, lines 506 507 def _shortcmd(self, line): 508 """Internal: send a command and get the response. 509 Same return value as _getresp().""" 510 self._putcmd(line) 511 return self._getresp() 512 513 def _longcmd(self, line, file=None): 514 """Internal: send a command and get the response plus following text. 515 Same return value as _getlongresp().""" 516 self._putcmd(line) 517 return self._getlongresp(file) 518 519 def _longcmdstring(self, line, file=None): 520 """Internal: send a command and get the response plus following text. 521 Same as _longcmd() and _getlongresp(), except that the returned `lines` 522 are unicode strings rather than bytes objects. 523 """ 524 self._putcmd(line) 525 resp, list = self._getlongresp(file) 526 return resp, [line.decode(self.encoding, self.errors) 527 for line in list] 528 529 def _getoverviewfmt(self): 530 """Internal: get the overview format. Queries the server if not 531 already done, else returns the cached value.""" 532 try: 533 return self._cachedoverviewfmt 534 except AttributeError: 535 pass 536 try: 537 resp, lines = self._longcmdstring("LIST OVERVIEW.FMT") 538 except NNTPPermanentError: 539 # Not supported by server? 540 fmt = _DEFAULT_OVERVIEW_FMT[:] 541 else: 542 fmt = _parse_overview_fmt(lines) 543 self._cachedoverviewfmt = fmt 544 return fmt 545 546 def _grouplist(self, lines): 547 # Parse lines into "group last first flag" 548 return [GroupInfo(*line.split()) for line in lines] 549 550 def capabilities(self): 551 """Process a CAPABILITIES command. Not supported by all servers. 552 Return: 553 - resp: server response if successful 554 - caps: a dictionary mapping capability names to lists of tokens 555 (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] }) 556 """ 557 caps = {} 558 resp, lines = self._longcmdstring("CAPABILITIES") 559 for line in lines: 560 name, *tokens = line.split() 561 caps[name] = tokens 562 return resp, caps 563 564 def newgroups(self, date, *, file=None): 565 """Process a NEWGROUPS command. Arguments: 566 - date: a date or datetime object 567 Return: 568 - resp: server response if successful 569 - list: list of newsgroup names 570 """ 571 if not isinstance(date, (datetime.date, datetime.date)): 572 raise TypeError( 573 "the date parameter must be a date or datetime object, " 574 "not '{:40}'".format(date.__class__.__name__)) 575 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) 576 cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str) 577 resp, lines = self._longcmdstring(cmd, file) 578 return resp, self._grouplist(lines) 579 580 def newnews(self, group, date, *, file=None): 581 """Process a NEWNEWS command. Arguments: 582 - group: group name or '*' 583 - date: a date or datetime object 584 Return: 585 - resp: server response if successful 586 - list: list of message ids 587 """ 588 if not isinstance(date, (datetime.date, datetime.date)): 589 raise TypeError( 590 "the date parameter must be a date or datetime object, " 591 "not '{:40}'".format(date.__class__.__name__)) 592 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) 593 cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str) 594 return self._longcmdstring(cmd, file) 595 596 def list(self, group_pattern=None, *, file=None): 597 """Process a LIST or LIST ACTIVE command. Arguments: 598 - group_pattern: a pattern indicating which groups to query 599 - file: Filename string or file object to store the result in 600 Returns: 601 - resp: server response if successful 602 - list: list of (group, last, first, flag) (strings) 603 """ 604 if group_pattern is not None: 605 command = 'LIST ACTIVE ' + group_pattern 606 else: 607 command = 'LIST' 608 resp, lines = self._longcmdstring(command, file) 609 return resp, self._grouplist(lines) 610 611 def _getdescriptions(self, group_pattern, return_all): 612 line_pat = re.compile('^(?P<group>[^ \t]+)[ \t]+(.*)$') 613 # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first 614 resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern) 615 if not resp.startswith('215'): 616 # Now the deprecated XGTITLE. This either raises an error 617 # or succeeds with the same output structure as LIST 618 # NEWSGROUPS. 619 resp, lines = self._longcmdstring('XGTITLE ' + group_pattern) 620 groups = {} 621 for raw_line in lines: 622 match = line_pat.search(raw_line.strip()) 623 if match: 624 name, desc = match.group(1, 2) 625 if not return_all: 626 return desc 627 groups[name] = desc 628 if return_all: 629 return resp, groups 630 else: 631 # Nothing found 632 return '' 633 634 def description(self, group): 635 """Get a description for a single group. If more than one 636 group matches ('group' is a pattern), return the first. If no 637 group matches, return an empty string. 638 639 This elides the response code from the server, since it can 640 only be '215' or '285' (for xgtitle) anyway. If the response 641 code is needed, use the 'descriptions' method. 642 643 NOTE: This neither checks for a wildcard in 'group' nor does 644 it check whether the group actually exists.""" 645 return self._getdescriptions(group, False) 646 647 def descriptions(self, group_pattern): 648 """Get descriptions for a range of groups.""" 649 return self._getdescriptions(group_pattern, True) 650 651 def group(self, name): 652 """Process a GROUP command. Argument: 653 - group: the group name 654 Returns: 655 - resp: server response if successful 656 - count: number of articles 657 - first: first article number 658 - last: last article number 659 - name: the group name 660 """ 661 resp = self._shortcmd('GROUP ' + name) 662 if not resp.startswith('211'): 663 raise NNTPReplyError(resp) 664 words = resp.split() 665 count = first = last = 0 666 n = len(words) 667 if n > 1: 668 count = words[1] 669 if n > 2: 670 first = words[2] 671 if n > 3: 672 last = words[3] 673 if n > 4: 674 name = words[4].lower() 675 return resp, int(count), int(first), int(last), name 676 677 def help(self, *, file=None): 678 """Process a HELP command. Argument: 679 - file: Filename string or file object to store the result in 680 Returns: 681 - resp: server response if successful 682 - list: list of strings returned by the server in response to the 683 HELP command 684 """ 685 return self._longcmdstring('HELP', file) 686 687 def _statparse(self, resp): 688 """Internal: parse the response line of a STAT, NEXT, LAST, 689 ARTICLE, HEAD or BODY command.""" 690 if not resp.startswith('22'): 691 raise NNTPReplyError(resp) 692 words = resp.split() 693 art_num = int(words[1]) 694 message_id = words[2] 695 return resp, art_num, message_id 696 697 def _statcmd(self, line): 698 """Internal: process a STAT, NEXT or LAST command.""" 699 resp = self._shortcmd(line) 700 return self._statparse(resp) 701 702 def stat(self, message_spec=None): 703 """Process a STAT command. Argument: 704 - message_spec: article number or message id (if not specified, 705 the current article is selected) 706 Returns: 707 - resp: server response if successful 708 - art_num: the article number 709 - message_id: the message id 710 """ 711 if message_spec: 712 return self._statcmd('STAT {0}'.format(message_spec)) 713 else: 714 return self._statcmd('STAT') 715 716 def next(self): 717 """Process a NEXT command. No arguments. Return as for STAT.""" 718 return self._statcmd('NEXT') 719 720 def last(self): 721 """Process a LAST command. No arguments. Return as for STAT.""" 722 return self._statcmd('LAST') 723 724 def _artcmd(self, line, file=None): 725 """Internal: process a HEAD, BODY or ARTICLE command.""" 726 resp, lines = self._longcmd(line, file) 727 resp, art_num, message_id = self._statparse(resp) 728 return resp, ArticleInfo(art_num, message_id, lines) 729 730 def head(self, message_spec=None, *, file=None): 731 """Process a HEAD command. Argument: 732 - message_spec: article number or message id 733 - file: filename string or file object to store the headers in 734 Returns: 735 - resp: server response if successful 736 - ArticleInfo: (article number, message id, list of header lines) 737 """ 738 if message_spec is not None: 739 cmd = 'HEAD {0}'.format(message_spec) 740 else: 741 cmd = 'HEAD' 742 return self._artcmd(cmd, file) 743 744 def body(self, message_spec=None, *, file=None): 745 """Process a BODY command. Argument: 746 - message_spec: article number or message id 747 - file: filename string or file object to store the body in 748 Returns: 749 - resp: server response if successful 750 - ArticleInfo: (article number, message id, list of body lines) 751 """ 752 if message_spec is not None: 753 cmd = 'BODY {0}'.format(message_spec) 754 else: 755 cmd = 'BODY' 756 return self._artcmd(cmd, file) 757 758 def article(self, message_spec=None, *, file=None): 759 """Process an ARTICLE command. Argument: 760 - message_spec: article number or message id 761 - file: filename string or file object to store the article in 762 Returns: 763 - resp: server response if successful 764 - ArticleInfo: (article number, message id, list of article lines) 765 """ 766 if message_spec is not None: 767 cmd = 'ARTICLE {0}'.format(message_spec) 768 else: 769 cmd = 'ARTICLE' 770 return self._artcmd(cmd, file) 771 772 def slave(self): 773 """Process a SLAVE command. Returns: 774 - resp: server response if successful 775 """ 776 return self._shortcmd('SLAVE') 777 778 def xhdr(self, hdr, str, *, file=None): 779 """Process an XHDR command (optional server extension). Arguments: 780 - hdr: the header type (e.g. 'subject') 781 - str: an article nr, a message id, or a range nr1-nr2 782 - file: Filename string or file object to store the result in 783 Returns: 784 - resp: server response if successful 785 - list: list of (nr, value) strings 786 """ 787 pat = re.compile('^([0-9]+) ?(.*)\n?') 788 resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file) 789 def remove_number(line): 790 m = pat.match(line) 791 return m.group(1, 2) if m else line 792 return resp, [remove_number(line) for line in lines] 793 794 def xover(self, start, end, *, file=None): 795 """Process an XOVER command (optional server extension) Arguments: 796 - start: start of range 797 - end: end of range 798 - file: Filename string or file object to store the result in 799 Returns: 800 - resp: server response if successful 801 - list: list of dicts containing the response fields 802 """ 803 resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end), 804 file) 805 fmt = self._getoverviewfmt() 806 return resp, _parse_overview(lines, fmt) 807 808 def over(self, message_spec, *, file=None): 809 """Process an OVER command. If the command isn't supported, fall 810 back to XOVER. Arguments: 811 - message_spec: 812 - either a message id, indicating the article to fetch 813 information about 814 - or a (start, end) tuple, indicating a range of article numbers; 815 if end is None, information up to the newest message will be 816 retrieved 817 - or None, indicating the current article number must be used 818 - file: Filename string or file object to store the result in 819 Returns: 820 - resp: server response if successful 821 - list: list of dicts containing the response fields 822 823 NOTE: the "message id" form isn't supported by XOVER 824 """ 825 cmd = 'OVER' if 'OVER' in self._caps else 'XOVER' 826 if isinstance(message_spec, (tuple, list)): 827 start, end = message_spec 828 cmd += ' {0}-{1}'.format(start, end or '') 829 elif message_spec is not None: 830 cmd = cmd + ' ' + message_spec 831 resp, lines = self._longcmdstring(cmd, file) 832 fmt = self._getoverviewfmt() 833 return resp, _parse_overview(lines, fmt) 834 835 def xgtitle(self, group, *, file=None): 836 """Process an XGTITLE command (optional server extension) Arguments: 837 - group: group name wildcard (i.e. news.*) 838 Returns: 839 - resp: server response if successful 840 - list: list of (name,title) strings""" 841 warnings.warn("The XGTITLE extension is not actively used, " 842 "use descriptions() instead", 843 DeprecationWarning, 2) 844 line_pat = re.compile('^([^ \t]+)[ \t]+(.*)$') 845 resp, raw_lines = self._longcmdstring('XGTITLE ' + group, file) 846 lines = [] 847 for raw_line in raw_lines: 848 match = line_pat.search(raw_line.strip()) 849 if match: 850 lines.append(match.group(1, 2)) 851 return resp, lines 852 853 def xpath(self, id): 854 """Process an XPATH command (optional server extension) Arguments: 855 - id: Message id of article 856 Returns: 857 resp: server response if successful 858 path: directory path to article 859 """ 860 warnings.warn("The XPATH extension is not actively used", 861 DeprecationWarning, 2) 862 863 resp = self._shortcmd('XPATH {0}'.format(id)) 864 if not resp.startswith('223'): 865 raise NNTPReplyError(resp) 866 try: 867 [resp_num, path] = resp.split() 868 except ValueError: 869 raise NNTPReplyError(resp) 870 else: 871 return resp, path 872 873 def date(self): 874 """Process the DATE command. 875 Returns: 876 - resp: server response if successful 877 - date: datetime object 878 """ 879 resp = self._shortcmd("DATE") 880 if not resp.startswith('111'): 881 raise NNTPReplyError(resp) 882 elem = resp.split() 883 if len(elem) != 2: 884 raise NNTPDataError(resp) 885 date = elem[1] 886 if len(date) != 14: 887 raise NNTPDataError(resp) 888 return resp, _parse_datetime(date, None) 889 890 def _post(self, command, f): 891 resp = self._shortcmd(command) 892 # Raises a specific exception if posting is not allowed 893 if not resp.startswith('3'): 894 raise NNTPReplyError(resp) 895 if isinstance(f, (bytes, bytearray)): 896 f = f.splitlines() 897 # We don't use _putline() because: 898 # - we don't want additional CRLF if the file or iterable is already 899 # in the right format 900 # - we don't want a spurious flush() after each line is written 901 for line in f: 902 if not line.endswith(_CRLF): 903 line = line.rstrip(b"\r\n") + _CRLF 904 if line.startswith(b'.'): 905 line = b'.' + line 906 self.file.write(line) 907 self.file.write(b".\r\n") 908 self.file.flush() 909 return self._getresp() 910 911 def post(self, data): 912 """Process a POST command. Arguments: 913 - data: bytes object, iterable or file containing the article 914 Returns: 915 - resp: server response if successful""" 916 return self._post('POST', data) 917 918 def ihave(self, message_id, data): 919 """Process an IHAVE command. Arguments: 920 - message_id: message-id of the article 921 - data: file containing the article 922 Returns: 923 - resp: server response if successful 924 Note that if the server refuses the article an exception is raised.""" 925 return self._post('IHAVE {0}'.format(message_id), data) 926 927 def _close(self): 928 self.file.close() 929 del self.file 930 931 def quit(self): 932 """Process a QUIT command and close the socket. Returns: 933 - resp: server response if successful""" 934 try: 935 resp = self._shortcmd('QUIT') 936 finally: 937 self._close() 938 return resp 939 940 def login(self, user=None, password=None, usenetrc=True): 941 if self.authenticated: 942 raise ValueError("Already logged in.") 943 if not user and not usenetrc: 944 raise ValueError( 945 "At least one of `user` and `usenetrc` must be specified") 946 # If no login/password was specified but netrc was requested, 947 # try to get them from ~/.netrc 948 # Presume that if .netrc has an entry, NNRP authentication is required. 949 try: 950 if usenetrc and not user: 951 import netrc 952 credentials = netrc.netrc() 953 auth = credentials.authenticators(self.host) 954 if auth: 955 user = auth[0] 956 password = auth[2] 957 except OSError: 958 pass 959 # Perform NNTP authentication if needed. 960 if not user: 961 return 962 resp = self._shortcmd('authinfo user ' + user) 963 if resp.startswith('381'): 964 if not password: 965 raise NNTPReplyError(resp) 966 else: 967 resp = self._shortcmd('authinfo pass ' + password) 968 if not resp.startswith('281'): 969 raise NNTPPermanentError(resp) 970 # Capabilities might have changed after login 971 self._caps = None 972 self.getcapabilities() 973 # Attempt to send mode reader if it was requested after login. 974 # Only do so if we're not in reader mode already. 975 if self.readermode_afterauth and 'READER' not in self._caps: 976 self._setreadermode() 977 # Capabilities might have changed after MODE READER 978 self._caps = None 979 self.getcapabilities() 980 981 def _setreadermode(self): 982 try: 983 self.welcome = self._shortcmd('mode reader') 984 except NNTPPermanentError: 985 # Error 5xx, probably 'not implemented' 986 pass 987 except NNTPTemporaryError as e: 988 if e.response.startswith('480'): 989 # Need authorization before 'mode reader' 990 self.readermode_afterauth = True 991 else: 992 raise 993 994 if _have_ssl: 995 def starttls(self, context=None): 996 """Process a STARTTLS command. Arguments: 997 - context: SSL context to use for the encrypted connection 998 """ 999 # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if 1000 # a TLS session already exists. 1001 if self.tls_on: 1002 raise ValueError("TLS is already enabled.") 1003 if self.authenticated: 1004 raise ValueError("TLS cannot be started after authentication.") 1005 resp = self._shortcmd('STARTTLS') 1006 if resp.startswith('382'): 1007 self.file.close() 1008 self.sock = _encrypt_on(self.sock, context, self.host) 1009 self.file = self.sock.makefile("rwb") 1010 self.tls_on = True 1011 # Capabilities may change after TLS starts up, so ask for them 1012 # again. 1013 self._caps = None 1014 self.getcapabilities() 1015 else: 1016 raise NNTPError("TLS failed to start.") 1017 1018 1019class NNTP(_NNTPBase): 1020 1021 def __init__(self, host, port=NNTP_PORT, user=None, password=None, 1022 readermode=None, usenetrc=False, 1023 timeout=_GLOBAL_DEFAULT_TIMEOUT): 1024 """Initialize an instance. Arguments: 1025 - host: hostname to connect to 1026 - port: port to connect to (default the standard NNTP port) 1027 - user: username to authenticate with 1028 - password: password to use with username 1029 - readermode: if true, send 'mode reader' command after 1030 connecting. 1031 - usenetrc: allow loading username and password from ~/.netrc file 1032 if not specified explicitly 1033 - timeout: timeout (in seconds) used for socket connections 1034 1035 readermode is sometimes necessary if you are connecting to an 1036 NNTP server on the local machine and intend to call 1037 reader-specific commands, such as `group'. If you get 1038 unexpected NNTPPermanentErrors, you might need to set 1039 readermode. 1040 """ 1041 self.host = host 1042 self.port = port 1043 self.sock = socket.create_connection((host, port), timeout) 1044 file = None 1045 try: 1046 file = self.sock.makefile("rwb") 1047 _NNTPBase.__init__(self, file, host, 1048 readermode, timeout) 1049 if user or usenetrc: 1050 self.login(user, password, usenetrc) 1051 except: 1052 if file: 1053 file.close() 1054 self.sock.close() 1055 raise 1056 1057 def _close(self): 1058 try: 1059 _NNTPBase._close(self) 1060 finally: 1061 self.sock.close() 1062 1063 1064if _have_ssl: 1065 class NNTP_SSL(_NNTPBase): 1066 1067 def __init__(self, host, port=NNTP_SSL_PORT, 1068 user=None, password=None, ssl_context=None, 1069 readermode=None, usenetrc=False, 1070 timeout=_GLOBAL_DEFAULT_TIMEOUT): 1071 """This works identically to NNTP.__init__, except for the change 1072 in default port and the `ssl_context` argument for SSL connections. 1073 """ 1074 self.sock = socket.create_connection((host, port), timeout) 1075 file = None 1076 try: 1077 self.sock = _encrypt_on(self.sock, ssl_context, host) 1078 file = self.sock.makefile("rwb") 1079 _NNTPBase.__init__(self, file, host, 1080 readermode=readermode, timeout=timeout) 1081 if user or usenetrc: 1082 self.login(user, password, usenetrc) 1083 except: 1084 if file: 1085 file.close() 1086 self.sock.close() 1087 raise 1088 1089 def _close(self): 1090 try: 1091 _NNTPBase._close(self) 1092 finally: 1093 self.sock.close() 1094 1095 __all__.append("NNTP_SSL") 1096 1097 1098# Test retrieval when run as a script. 1099if __name__ == '__main__': 1100 import argparse 1101 1102 parser = argparse.ArgumentParser(description="""\ 1103 nntplib built-in demo - display the latest articles in a newsgroup""") 1104 parser.add_argument('-g', '--group', default='gmane.comp.python.general', 1105 help='group to fetch messages from (default: %(default)s)') 1106 parser.add_argument('-s', '--server', default='news.gmane.org', 1107 help='NNTP server hostname (default: %(default)s)') 1108 parser.add_argument('-p', '--port', default=-1, type=int, 1109 help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT)) 1110 parser.add_argument('-n', '--nb-articles', default=10, type=int, 1111 help='number of articles to fetch (default: %(default)s)') 1112 parser.add_argument('-S', '--ssl', action='store_true', default=False, 1113 help='use NNTP over SSL') 1114 args = parser.parse_args() 1115 1116 port = args.port 1117 if not args.ssl: 1118 if port == -1: 1119 port = NNTP_PORT 1120 s = NNTP(host=args.server, port=port) 1121 else: 1122 if port == -1: 1123 port = NNTP_SSL_PORT 1124 s = NNTP_SSL(host=args.server, port=port) 1125 1126 caps = s.getcapabilities() 1127 if 'STARTTLS' in caps: 1128 s.starttls() 1129 resp, count, first, last, name = s.group(args.group) 1130 print('Group', name, 'has', count, 'articles, range', first, 'to', last) 1131 1132 def cut(s, lim): 1133 if len(s) > lim: 1134 s = s[:lim - 4] + "..." 1135 return s 1136 1137 first = str(int(last) - args.nb_articles + 1) 1138 resp, overviews = s.xover(first, last) 1139 for artnum, over in overviews: 1140 author = decode_header(over['from']).split('<', 1)[0] 1141 subject = decode_header(over['subject']) 1142 lines = int(over[':lines']) 1143 print("{:7} {:20} {:42} ({})".format( 1144 artnum, cut(author, 20), cut(subject, 42), lines) 1145 ) 1146 1147 s.quit() 1148