1 """An NNTP client class based on:
2 - RFC 977: Network News Transfer Protocol
3 - RFC 2980: Common NNTP Extensions
4 - RFC 3977: Network News Transfer Protocol (version 2)
5
6 Example:
7
8 >>> from nntplib import NNTP
9 >>> s = NNTP('news')
10 >>> resp, count, first, last, name = s.group('comp.lang.python')
11 >>> print('Group', name, 'has', count, 'articles, range', first, 'to', last)
12 Group comp.lang.python has 51 articles, range 5770 to 5821
13 >>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last))
14 >>> resp = s.quit()
15 >>>
16
17 Here 'resp' is the server response line.
18 Error responses are turned into exceptions.
19
20 To post an article from a file:
21 >>> f = open(filename, 'rb') # file containing article, including header
22 >>> resp = s.post(f)
23 >>>
24
25 For descriptions of all methods, read the comments in the code below.
26 Note that all arguments and return values representing article numbers
27 are strings, not numbers, since they are rarely used for calculations.
28 """
29
30 # RFC 977 by Brian Kantor and Phil Lapsley.
31 # xover, xgtitle, xpath, date methods by Kevan Heydon
32
33 # Incompatible changes from the 2.x nntplib:
34 # - all commands are encoded as UTF-8 data (using the "surrogateescape"
35 # error handler), except for raw message data (POST, IHAVE)
36 # - all responses are decoded as UTF-8 data (using the "surrogateescape"
37 # error handler), except for raw message data (ARTICLE, HEAD, BODY)
38 # - the `file` argument to various methods is keyword-only
39 #
40 # - NNTP.date() returns a datetime object
41 # - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object,
42 # rather than a pair of (date, time) strings.
43 # - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples
44 # - NNTP.descriptions() returns a dict mapping group names to descriptions
45 # - NNTP.xover() returns a list of dicts mapping field names (header or metadata)
46 # to field values; each dict representing a message overview.
47 # - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo)
48 # tuple.
49 # - the "internal" methods have been marked private (they now start with
50 # an underscore)
51
52 # Other changes from the 2.x/3.1 nntplib:
53 # - automatic querying of capabilities at connect
54 # - New method NNTP.getcapabilities()
55 # - New method NNTP.over()
56 # - New helper function decode_header()
57 # - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and
58 # arbitrary iterables yielding lines.
59 # - An extensive test suite :-)
60
61 # TODO:
62 # - return structured data (GroupInfo etc.) everywhere
63 # - support HDR
64
65 # Imports
66 import re
67 import socket
68 import collections
69 import datetime
70 import sys
71 import warnings
72
73 try:
74 import ssl
75 except ImportError:
76 _have_ssl = False
77 else:
78 _have_ssl = True
79
80 from email.header import decode_header as _email_decode_header
81 from socket import _GLOBAL_DEFAULT_TIMEOUT
82
83 __all__ = ["NNTP",
84 "NNTPError", "NNTPReplyError", "NNTPTemporaryError",
85 "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError",
86 "decode_header",
87 ]
88
89 warnings._deprecated(__name__, remove=(3, 13))
90
91 # maximal line length when calling readline(). This is to prevent
92 # reading arbitrary length lines. RFC 3977 limits NNTP line length to
93 # 512 characters, including CRLF. We have selected 2048 just to be on
94 # the safe side.
95 _MAXLINE = 2048
96
97
98 # Exceptions raised when an error or invalid response is received
99 class ESC[4;38;5;81mNNTPError(ESC[4;38;5;149mException):
100 """Base class for all nntplib exceptions"""
101 def __init__(self, *args):
102 Exception.__init__(self, *args)
103 try:
104 self.response = args[0]
105 except IndexError:
106 self.response = 'No response given'
107
108 class ESC[4;38;5;81mNNTPReplyError(ESC[4;38;5;149mNNTPError):
109 """Unexpected [123]xx reply"""
110 pass
111
112 class ESC[4;38;5;81mNNTPTemporaryError(ESC[4;38;5;149mNNTPError):
113 """4xx errors"""
114 pass
115
116 class ESC[4;38;5;81mNNTPPermanentError(ESC[4;38;5;149mNNTPError):
117 """5xx errors"""
118 pass
119
120 class ESC[4;38;5;81mNNTPProtocolError(ESC[4;38;5;149mNNTPError):
121 """Response does not begin with [1-5]"""
122 pass
123
124 class ESC[4;38;5;81mNNTPDataError(ESC[4;38;5;149mNNTPError):
125 """Error in response data"""
126 pass
127
128
129 # Standard port used by NNTP servers
130 NNTP_PORT = 119
131 NNTP_SSL_PORT = 563
132
133 # Response numbers that are followed by additional text (e.g. article)
134 _LONGRESP = {
135 '100', # HELP
136 '101', # CAPABILITIES
137 '211', # LISTGROUP (also not multi-line with GROUP)
138 '215', # LIST
139 '220', # ARTICLE
140 '221', # HEAD, XHDR
141 '222', # BODY
142 '224', # OVER, XOVER
143 '225', # HDR
144 '230', # NEWNEWS
145 '231', # NEWGROUPS
146 '282', # XGTITLE
147 }
148
149 # Default decoded value for LIST OVERVIEW.FMT if not supported
150 _DEFAULT_OVERVIEW_FMT = [
151 "subject", "from", "date", "message-id", "references", ":bytes", ":lines"]
152
153 # Alternative names allowed in LIST OVERVIEW.FMT response
154 _OVERVIEW_FMT_ALTERNATIVES = {
155 'bytes': ':bytes',
156 'lines': ':lines',
157 }
158
159 # Line terminators (we always output CRLF, but accept any of CRLF, CR, LF)
160 _CRLF = b'\r\n'
161
162 GroupInfo = collections.namedtuple('GroupInfo',
163 ['group', 'last', 'first', 'flag'])
164
165 ArticleInfo = collections.namedtuple('ArticleInfo',
166 ['number', 'message_id', 'lines'])
167
168
169 # Helper function(s)
170 def decode_header(header_str):
171 """Takes a unicode string representing a munged header value
172 and decodes it as a (possibly non-ASCII) readable value."""
173 parts = []
174 for v, enc in _email_decode_header(header_str):
175 if isinstance(v, bytes):
176 parts.append(v.decode(enc or 'ascii'))
177 else:
178 parts.append(v)
179 return ''.join(parts)
180
181 def _parse_overview_fmt(lines):
182 """Parse a list of string representing the response to LIST OVERVIEW.FMT
183 and return a list of header/metadata names.
184 Raises NNTPDataError if the response is not compliant
185 (cf. RFC 3977, section 8.4)."""
186 fmt = []
187 for line in lines:
188 if line[0] == ':':
189 # Metadata name (e.g. ":bytes")
190 name, _, suffix = line[1:].partition(':')
191 name = ':' + name
192 else:
193 # Header name (e.g. "Subject:" or "Xref:full")
194 name, _, suffix = line.partition(':')
195 name = name.lower()
196 name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name)
197 # Should we do something with the suffix?
198 fmt.append(name)
199 defaults = _DEFAULT_OVERVIEW_FMT
200 if len(fmt) < len(defaults):
201 raise NNTPDataError("LIST OVERVIEW.FMT response too short")
202 if fmt[:len(defaults)] != defaults:
203 raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields")
204 return fmt
205
206 def _parse_overview(lines, fmt, data_process_func=None):
207 """Parse the response to an OVER or XOVER command according to the
208 overview format `fmt`."""
209 n_defaults = len(_DEFAULT_OVERVIEW_FMT)
210 overview = []
211 for line in lines:
212 fields = {}
213 article_number, *tokens = line.split('\t')
214 article_number = int(article_number)
215 for i, token in enumerate(tokens):
216 if i >= len(fmt):
217 # XXX should we raise an error? Some servers might not
218 # support LIST OVERVIEW.FMT and still return additional
219 # headers.
220 continue
221 field_name = fmt[i]
222 is_metadata = field_name.startswith(':')
223 if i >= n_defaults and not is_metadata:
224 # Non-default header names are included in full in the response
225 # (unless the field is totally empty)
226 h = field_name + ": "
227 if token and token[:len(h)].lower() != h:
228 raise NNTPDataError("OVER/XOVER response doesn't include "
229 "names of additional headers")
230 token = token[len(h):] if token else None
231 fields[fmt[i]] = token
232 overview.append((article_number, fields))
233 return overview
234
235 def _parse_datetime(date_str, time_str=None):
236 """Parse a pair of (date, time) strings, and return a datetime object.
237 If only the date is given, it is assumed to be date and time
238 concatenated together (e.g. response to the DATE command).
239 """
240 if time_str is None:
241 time_str = date_str[-6:]
242 date_str = date_str[:-6]
243 hours = int(time_str[:2])
244 minutes = int(time_str[2:4])
245 seconds = int(time_str[4:])
246 year = int(date_str[:-4])
247 month = int(date_str[-4:-2])
248 day = int(date_str[-2:])
249 # RFC 3977 doesn't say how to interpret 2-char years. Assume that
250 # there are no dates before 1970 on Usenet.
251 if year < 70:
252 year += 2000
253 elif year < 100:
254 year += 1900
255 return datetime.datetime(year, month, day, hours, minutes, seconds)
256
257 def _unparse_datetime(dt, legacy=False):
258 """Format a date or datetime object as a pair of (date, time) strings
259 in the format required by the NEWNEWS and NEWGROUPS commands. If a
260 date object is passed, the time is assumed to be midnight (00h00).
261
262 The returned representation depends on the legacy flag:
263 * if legacy is False (the default):
264 date has the YYYYMMDD format and time the HHMMSS format
265 * if legacy is True:
266 date has the YYMMDD format and time the HHMMSS format.
267 RFC 3977 compliant servers should understand both formats; therefore,
268 legacy is only needed when talking to old servers.
269 """
270 if not isinstance(dt, datetime.datetime):
271 time_str = "000000"
272 else:
273 time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt)
274 y = dt.year
275 if legacy:
276 y = y % 100
277 date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt)
278 else:
279 date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt)
280 return date_str, time_str
281
282
283 if _have_ssl:
284
285 def _encrypt_on(sock, context, hostname):
286 """Wrap a socket in SSL/TLS. Arguments:
287 - sock: Socket to wrap
288 - context: SSL context to use for the encrypted connection
289 Returns:
290 - sock: New, encrypted socket.
291 """
292 # Generate a default SSL context if none was passed.
293 if context is None:
294 context = ssl._create_stdlib_context()
295 return context.wrap_socket(sock, server_hostname=hostname)
296
297
298 # The classes themselves
299 class ESC[4;38;5;81mNNTP:
300 # UTF-8 is the character set for all NNTP commands and responses: they
301 # are automatically encoded (when sending) and decoded (and receiving)
302 # by this class.
303 # However, some multi-line data blocks can contain arbitrary bytes (for
304 # example, latin-1 or utf-16 data in the body of a message). Commands
305 # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message
306 # data will therefore only accept and produce bytes objects.
307 # Furthermore, since there could be non-compliant servers out there,
308 # we use 'surrogateescape' as the error handler for fault tolerance
309 # and easy round-tripping. This could be useful for some applications
310 # (e.g. NNTP gateways).
311
312 encoding = 'utf-8'
313 errors = 'surrogateescape'
314
315 def __init__(self, host, port=NNTP_PORT, user=None, password=None,
316 readermode=None, usenetrc=False,
317 timeout=_GLOBAL_DEFAULT_TIMEOUT):
318 """Initialize an instance. Arguments:
319 - host: hostname to connect to
320 - port: port to connect to (default the standard NNTP port)
321 - user: username to authenticate with
322 - password: password to use with username
323 - readermode: if true, send 'mode reader' command after
324 connecting.
325 - usenetrc: allow loading username and password from ~/.netrc file
326 if not specified explicitly
327 - timeout: timeout (in seconds) used for socket connections
328
329 readermode is sometimes necessary if you are connecting to an
330 NNTP server on the local machine and intend to call
331 reader-specific commands, such as `group'. If you get
332 unexpected NNTPPermanentErrors, you might need to set
333 readermode.
334 """
335 self.host = host
336 self.port = port
337 self.sock = self._create_socket(timeout)
338 self.file = None
339 try:
340 self.file = self.sock.makefile("rwb")
341 self._base_init(readermode)
342 if user or usenetrc:
343 self.login(user, password, usenetrc)
344 except:
345 if self.file:
346 self.file.close()
347 self.sock.close()
348 raise
349
350 def _base_init(self, readermode):
351 """Partial initialization for the NNTP protocol.
352 This instance method is extracted for supporting the test code.
353 """
354 self.debugging = 0
355 self.welcome = self._getresp()
356
357 # Inquire about capabilities (RFC 3977).
358 self._caps = None
359 self.getcapabilities()
360
361 # 'MODE READER' is sometimes necessary to enable 'reader' mode.
362 # However, the order in which 'MODE READER' and 'AUTHINFO' need to
363 # arrive differs between some NNTP servers. If _setreadermode() fails
364 # with an authorization failed error, it will set this to True;
365 # the login() routine will interpret that as a request to try again
366 # after performing its normal function.
367 # Enable only if we're not already in READER mode anyway.
368 self.readermode_afterauth = False
369 if readermode and 'READER' not in self._caps:
370 self._setreadermode()
371 if not self.readermode_afterauth:
372 # Capabilities might have changed after MODE READER
373 self._caps = None
374 self.getcapabilities()
375
376 # RFC 4642 2.2.2: Both the client and the server MUST know if there is
377 # a TLS session active. A client MUST NOT attempt to start a TLS
378 # session if a TLS session is already active.
379 self.tls_on = False
380
381 # Log in and encryption setup order is left to subclasses.
382 self.authenticated = False
383
384 def __enter__(self):
385 return self
386
387 def __exit__(self, *args):
388 is_connected = lambda: hasattr(self, "file")
389 if is_connected():
390 try:
391 self.quit()
392 except (OSError, EOFError):
393 pass
394 finally:
395 if is_connected():
396 self._close()
397
398 def _create_socket(self, timeout):
399 if timeout is not None and not timeout:
400 raise ValueError('Non-blocking socket (timeout=0) is not supported')
401 sys.audit("nntplib.connect", self, self.host, self.port)
402 return socket.create_connection((self.host, self.port), timeout)
403
404 def getwelcome(self):
405 """Get the welcome message from the server
406 (this is read and squirreled away by __init__()).
407 If the response code is 200, posting is allowed;
408 if it 201, posting is not allowed."""
409
410 if self.debugging: print('*welcome*', repr(self.welcome))
411 return self.welcome
412
413 def getcapabilities(self):
414 """Get the server capabilities, as read by __init__().
415 If the CAPABILITIES command is not supported, an empty dict is
416 returned."""
417 if self._caps is None:
418 self.nntp_version = 1
419 self.nntp_implementation = None
420 try:
421 resp, caps = self.capabilities()
422 except (NNTPPermanentError, NNTPTemporaryError):
423 # Server doesn't support capabilities
424 self._caps = {}
425 else:
426 self._caps = caps
427 if 'VERSION' in caps:
428 # The server can advertise several supported versions,
429 # choose the highest.
430 self.nntp_version = max(map(int, caps['VERSION']))
431 if 'IMPLEMENTATION' in caps:
432 self.nntp_implementation = ' '.join(caps['IMPLEMENTATION'])
433 return self._caps
434
435 def set_debuglevel(self, level):
436 """Set the debugging level. Argument 'level' means:
437 0: no debugging output (default)
438 1: print commands and responses but not body text etc.
439 2: also print raw lines read and sent before stripping CR/LF"""
440
441 self.debugging = level
442 debug = set_debuglevel
443
444 def _putline(self, line):
445 """Internal: send one line to the server, appending CRLF.
446 The `line` must be a bytes-like object."""
447 sys.audit("nntplib.putline", self, line)
448 line = line + _CRLF
449 if self.debugging > 1: print('*put*', repr(line))
450 self.file.write(line)
451 self.file.flush()
452
453 def _putcmd(self, line):
454 """Internal: send one command to the server (through _putline()).
455 The `line` must be a unicode string."""
456 if self.debugging: print('*cmd*', repr(line))
457 line = line.encode(self.encoding, self.errors)
458 self._putline(line)
459
460 def _getline(self, strip_crlf=True):
461 """Internal: return one line from the server, stripping _CRLF.
462 Raise EOFError if the connection is closed.
463 Returns a bytes object."""
464 line = self.file.readline(_MAXLINE +1)
465 if len(line) > _MAXLINE:
466 raise NNTPDataError('line too long')
467 if self.debugging > 1:
468 print('*get*', repr(line))
469 if not line: raise EOFError
470 if strip_crlf:
471 if line[-2:] == _CRLF:
472 line = line[:-2]
473 elif line[-1:] in _CRLF:
474 line = line[:-1]
475 return line
476
477 def _getresp(self):
478 """Internal: get a response from the server.
479 Raise various errors if the response indicates an error.
480 Returns a unicode string."""
481 resp = self._getline()
482 if self.debugging: print('*resp*', repr(resp))
483 resp = resp.decode(self.encoding, self.errors)
484 c = resp[:1]
485 if c == '4':
486 raise NNTPTemporaryError(resp)
487 if c == '5':
488 raise NNTPPermanentError(resp)
489 if c not in '123':
490 raise NNTPProtocolError(resp)
491 return resp
492
493 def _getlongresp(self, file=None):
494 """Internal: get a response plus following text from the server.
495 Raise various errors if the response indicates an error.
496
497 Returns a (response, lines) tuple where `response` is a unicode
498 string and `lines` is a list of bytes objects.
499 If `file` is a file-like object, it must be open in binary mode.
500 """
501
502 openedFile = None
503 try:
504 # If a string was passed then open a file with that name
505 if isinstance(file, (str, bytes)):
506 openedFile = file = open(file, "wb")
507
508 resp = self._getresp()
509 if resp[:3] not in _LONGRESP:
510 raise NNTPReplyError(resp)
511
512 lines = []
513 if file is not None:
514 # XXX lines = None instead?
515 terminators = (b'.' + _CRLF, b'.\n')
516 while 1:
517 line = self._getline(False)
518 if line in terminators:
519 break
520 if line.startswith(b'..'):
521 line = line[1:]
522 file.write(line)
523 else:
524 terminator = b'.'
525 while 1:
526 line = self._getline()
527 if line == terminator:
528 break
529 if line.startswith(b'..'):
530 line = line[1:]
531 lines.append(line)
532 finally:
533 # If this method created the file, then it must close it
534 if openedFile:
535 openedFile.close()
536
537 return resp, lines
538
539 def _shortcmd(self, line):
540 """Internal: send a command and get the response.
541 Same return value as _getresp()."""
542 self._putcmd(line)
543 return self._getresp()
544
545 def _longcmd(self, line, file=None):
546 """Internal: send a command and get the response plus following text.
547 Same return value as _getlongresp()."""
548 self._putcmd(line)
549 return self._getlongresp(file)
550
551 def _longcmdstring(self, line, file=None):
552 """Internal: send a command and get the response plus following text.
553 Same as _longcmd() and _getlongresp(), except that the returned `lines`
554 are unicode strings rather than bytes objects.
555 """
556 self._putcmd(line)
557 resp, list = self._getlongresp(file)
558 return resp, [line.decode(self.encoding, self.errors)
559 for line in list]
560
561 def _getoverviewfmt(self):
562 """Internal: get the overview format. Queries the server if not
563 already done, else returns the cached value."""
564 try:
565 return self._cachedoverviewfmt
566 except AttributeError:
567 pass
568 try:
569 resp, lines = self._longcmdstring("LIST OVERVIEW.FMT")
570 except NNTPPermanentError:
571 # Not supported by server?
572 fmt = _DEFAULT_OVERVIEW_FMT[:]
573 else:
574 fmt = _parse_overview_fmt(lines)
575 self._cachedoverviewfmt = fmt
576 return fmt
577
578 def _grouplist(self, lines):
579 # Parse lines into "group last first flag"
580 return [GroupInfo(*line.split()) for line in lines]
581
582 def capabilities(self):
583 """Process a CAPABILITIES command. Not supported by all servers.
584 Return:
585 - resp: server response if successful
586 - caps: a dictionary mapping capability names to lists of tokens
587 (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] })
588 """
589 caps = {}
590 resp, lines = self._longcmdstring("CAPABILITIES")
591 for line in lines:
592 name, *tokens = line.split()
593 caps[name] = tokens
594 return resp, caps
595
596 def newgroups(self, date, *, file=None):
597 """Process a NEWGROUPS command. Arguments:
598 - date: a date or datetime object
599 Return:
600 - resp: server response if successful
601 - list: list of newsgroup names
602 """
603 if not isinstance(date, (datetime.date, datetime.date)):
604 raise TypeError(
605 "the date parameter must be a date or datetime object, "
606 "not '{:40}'".format(date.__class__.__name__))
607 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2)
608 cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str)
609 resp, lines = self._longcmdstring(cmd, file)
610 return resp, self._grouplist(lines)
611
612 def newnews(self, group, date, *, file=None):
613 """Process a NEWNEWS command. Arguments:
614 - group: group name or '*'
615 - date: a date or datetime object
616 Return:
617 - resp: server response if successful
618 - list: list of message ids
619 """
620 if not isinstance(date, (datetime.date, datetime.date)):
621 raise TypeError(
622 "the date parameter must be a date or datetime object, "
623 "not '{:40}'".format(date.__class__.__name__))
624 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2)
625 cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str)
626 return self._longcmdstring(cmd, file)
627
628 def list(self, group_pattern=None, *, file=None):
629 """Process a LIST or LIST ACTIVE command. Arguments:
630 - group_pattern: a pattern indicating which groups to query
631 - file: Filename string or file object to store the result in
632 Returns:
633 - resp: server response if successful
634 - list: list of (group, last, first, flag) (strings)
635 """
636 if group_pattern is not None:
637 command = 'LIST ACTIVE ' + group_pattern
638 else:
639 command = 'LIST'
640 resp, lines = self._longcmdstring(command, file)
641 return resp, self._grouplist(lines)
642
643 def _getdescriptions(self, group_pattern, return_all):
644 line_pat = re.compile('^(?P<group>[^ \t]+)[ \t]+(.*)$')
645 # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first
646 resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern)
647 if not resp.startswith('215'):
648 # Now the deprecated XGTITLE. This either raises an error
649 # or succeeds with the same output structure as LIST
650 # NEWSGROUPS.
651 resp, lines = self._longcmdstring('XGTITLE ' + group_pattern)
652 groups = {}
653 for raw_line in lines:
654 match = line_pat.search(raw_line.strip())
655 if match:
656 name, desc = match.group(1, 2)
657 if not return_all:
658 return desc
659 groups[name] = desc
660 if return_all:
661 return resp, groups
662 else:
663 # Nothing found
664 return ''
665
666 def description(self, group):
667 """Get a description for a single group. If more than one
668 group matches ('group' is a pattern), return the first. If no
669 group matches, return an empty string.
670
671 This elides the response code from the server, since it can
672 only be '215' or '285' (for xgtitle) anyway. If the response
673 code is needed, use the 'descriptions' method.
674
675 NOTE: This neither checks for a wildcard in 'group' nor does
676 it check whether the group actually exists."""
677 return self._getdescriptions(group, False)
678
679 def descriptions(self, group_pattern):
680 """Get descriptions for a range of groups."""
681 return self._getdescriptions(group_pattern, True)
682
683 def group(self, name):
684 """Process a GROUP command. Argument:
685 - group: the group name
686 Returns:
687 - resp: server response if successful
688 - count: number of articles
689 - first: first article number
690 - last: last article number
691 - name: the group name
692 """
693 resp = self._shortcmd('GROUP ' + name)
694 if not resp.startswith('211'):
695 raise NNTPReplyError(resp)
696 words = resp.split()
697 count = first = last = 0
698 n = len(words)
699 if n > 1:
700 count = words[1]
701 if n > 2:
702 first = words[2]
703 if n > 3:
704 last = words[3]
705 if n > 4:
706 name = words[4].lower()
707 return resp, int(count), int(first), int(last), name
708
709 def help(self, *, file=None):
710 """Process a HELP command. Argument:
711 - file: Filename string or file object to store the result in
712 Returns:
713 - resp: server response if successful
714 - list: list of strings returned by the server in response to the
715 HELP command
716 """
717 return self._longcmdstring('HELP', file)
718
719 def _statparse(self, resp):
720 """Internal: parse the response line of a STAT, NEXT, LAST,
721 ARTICLE, HEAD or BODY command."""
722 if not resp.startswith('22'):
723 raise NNTPReplyError(resp)
724 words = resp.split()
725 art_num = int(words[1])
726 message_id = words[2]
727 return resp, art_num, message_id
728
729 def _statcmd(self, line):
730 """Internal: process a STAT, NEXT or LAST command."""
731 resp = self._shortcmd(line)
732 return self._statparse(resp)
733
734 def stat(self, message_spec=None):
735 """Process a STAT command. Argument:
736 - message_spec: article number or message id (if not specified,
737 the current article is selected)
738 Returns:
739 - resp: server response if successful
740 - art_num: the article number
741 - message_id: the message id
742 """
743 if message_spec:
744 return self._statcmd('STAT {0}'.format(message_spec))
745 else:
746 return self._statcmd('STAT')
747
748 def next(self):
749 """Process a NEXT command. No arguments. Return as for STAT."""
750 return self._statcmd('NEXT')
751
752 def last(self):
753 """Process a LAST command. No arguments. Return as for STAT."""
754 return self._statcmd('LAST')
755
756 def _artcmd(self, line, file=None):
757 """Internal: process a HEAD, BODY or ARTICLE command."""
758 resp, lines = self._longcmd(line, file)
759 resp, art_num, message_id = self._statparse(resp)
760 return resp, ArticleInfo(art_num, message_id, lines)
761
762 def head(self, message_spec=None, *, file=None):
763 """Process a HEAD command. Argument:
764 - message_spec: article number or message id
765 - file: filename string or file object to store the headers in
766 Returns:
767 - resp: server response if successful
768 - ArticleInfo: (article number, message id, list of header lines)
769 """
770 if message_spec is not None:
771 cmd = 'HEAD {0}'.format(message_spec)
772 else:
773 cmd = 'HEAD'
774 return self._artcmd(cmd, file)
775
776 def body(self, message_spec=None, *, file=None):
777 """Process a BODY command. Argument:
778 - message_spec: article number or message id
779 - file: filename string or file object to store the body in
780 Returns:
781 - resp: server response if successful
782 - ArticleInfo: (article number, message id, list of body lines)
783 """
784 if message_spec is not None:
785 cmd = 'BODY {0}'.format(message_spec)
786 else:
787 cmd = 'BODY'
788 return self._artcmd(cmd, file)
789
790 def article(self, message_spec=None, *, file=None):
791 """Process an ARTICLE command. Argument:
792 - message_spec: article number or message id
793 - file: filename string or file object to store the article in
794 Returns:
795 - resp: server response if successful
796 - ArticleInfo: (article number, message id, list of article lines)
797 """
798 if message_spec is not None:
799 cmd = 'ARTICLE {0}'.format(message_spec)
800 else:
801 cmd = 'ARTICLE'
802 return self._artcmd(cmd, file)
803
804 def slave(self):
805 """Process a SLAVE command. Returns:
806 - resp: server response if successful
807 """
808 return self._shortcmd('SLAVE')
809
810 def xhdr(self, hdr, str, *, file=None):
811 """Process an XHDR command (optional server extension). Arguments:
812 - hdr: the header type (e.g. 'subject')
813 - str: an article nr, a message id, or a range nr1-nr2
814 - file: Filename string or file object to store the result in
815 Returns:
816 - resp: server response if successful
817 - list: list of (nr, value) strings
818 """
819 pat = re.compile('^([0-9]+) ?(.*)\n?')
820 resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file)
821 def remove_number(line):
822 m = pat.match(line)
823 return m.group(1, 2) if m else line
824 return resp, [remove_number(line) for line in lines]
825
826 def xover(self, start, end, *, file=None):
827 """Process an XOVER command (optional server extension) Arguments:
828 - start: start of range
829 - end: end of range
830 - file: Filename string or file object to store the result in
831 Returns:
832 - resp: server response if successful
833 - list: list of dicts containing the response fields
834 """
835 resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end),
836 file)
837 fmt = self._getoverviewfmt()
838 return resp, _parse_overview(lines, fmt)
839
840 def over(self, message_spec, *, file=None):
841 """Process an OVER command. If the command isn't supported, fall
842 back to XOVER. Arguments:
843 - message_spec:
844 - either a message id, indicating the article to fetch
845 information about
846 - or a (start, end) tuple, indicating a range of article numbers;
847 if end is None, information up to the newest message will be
848 retrieved
849 - or None, indicating the current article number must be used
850 - file: Filename string or file object to store the result in
851 Returns:
852 - resp: server response if successful
853 - list: list of dicts containing the response fields
854
855 NOTE: the "message id" form isn't supported by XOVER
856 """
857 cmd = 'OVER' if 'OVER' in self._caps else 'XOVER'
858 if isinstance(message_spec, (tuple, list)):
859 start, end = message_spec
860 cmd += ' {0}-{1}'.format(start, end or '')
861 elif message_spec is not None:
862 cmd = cmd + ' ' + message_spec
863 resp, lines = self._longcmdstring(cmd, file)
864 fmt = self._getoverviewfmt()
865 return resp, _parse_overview(lines, fmt)
866
867 def date(self):
868 """Process the DATE command.
869 Returns:
870 - resp: server response if successful
871 - date: datetime object
872 """
873 resp = self._shortcmd("DATE")
874 if not resp.startswith('111'):
875 raise NNTPReplyError(resp)
876 elem = resp.split()
877 if len(elem) != 2:
878 raise NNTPDataError(resp)
879 date = elem[1]
880 if len(date) != 14:
881 raise NNTPDataError(resp)
882 return resp, _parse_datetime(date, None)
883
884 def _post(self, command, f):
885 resp = self._shortcmd(command)
886 # Raises a specific exception if posting is not allowed
887 if not resp.startswith('3'):
888 raise NNTPReplyError(resp)
889 if isinstance(f, (bytes, bytearray)):
890 f = f.splitlines()
891 # We don't use _putline() because:
892 # - we don't want additional CRLF if the file or iterable is already
893 # in the right format
894 # - we don't want a spurious flush() after each line is written
895 for line in f:
896 if not line.endswith(_CRLF):
897 line = line.rstrip(b"\r\n") + _CRLF
898 if line.startswith(b'.'):
899 line = b'.' + line
900 self.file.write(line)
901 self.file.write(b".\r\n")
902 self.file.flush()
903 return self._getresp()
904
905 def post(self, data):
906 """Process a POST command. Arguments:
907 - data: bytes object, iterable or file containing the article
908 Returns:
909 - resp: server response if successful"""
910 return self._post('POST', data)
911
912 def ihave(self, message_id, data):
913 """Process an IHAVE command. Arguments:
914 - message_id: message-id of the article
915 - data: file containing the article
916 Returns:
917 - resp: server response if successful
918 Note that if the server refuses the article an exception is raised."""
919 return self._post('IHAVE {0}'.format(message_id), data)
920
921 def _close(self):
922 try:
923 if self.file:
924 self.file.close()
925 del self.file
926 finally:
927 self.sock.close()
928
929 def quit(self):
930 """Process a QUIT command and close the socket. Returns:
931 - resp: server response if successful"""
932 try:
933 resp = self._shortcmd('QUIT')
934 finally:
935 self._close()
936 return resp
937
938 def login(self, user=None, password=None, usenetrc=True):
939 if self.authenticated:
940 raise ValueError("Already logged in.")
941 if not user and not usenetrc:
942 raise ValueError(
943 "At least one of `user` and `usenetrc` must be specified")
944 # If no login/password was specified but netrc was requested,
945 # try to get them from ~/.netrc
946 # Presume that if .netrc has an entry, NNRP authentication is required.
947 try:
948 if usenetrc and not user:
949 import netrc
950 credentials = netrc.netrc()
951 auth = credentials.authenticators(self.host)
952 if auth:
953 user = auth[0]
954 password = auth[2]
955 except OSError:
956 pass
957 # Perform NNTP authentication if needed.
958 if not user:
959 return
960 resp = self._shortcmd('authinfo user ' + user)
961 if resp.startswith('381'):
962 if not password:
963 raise NNTPReplyError(resp)
964 else:
965 resp = self._shortcmd('authinfo pass ' + password)
966 if not resp.startswith('281'):
967 raise NNTPPermanentError(resp)
968 # Capabilities might have changed after login
969 self._caps = None
970 self.getcapabilities()
971 # Attempt to send mode reader if it was requested after login.
972 # Only do so if we're not in reader mode already.
973 if self.readermode_afterauth and 'READER' not in self._caps:
974 self._setreadermode()
975 # Capabilities might have changed after MODE READER
976 self._caps = None
977 self.getcapabilities()
978
979 def _setreadermode(self):
980 try:
981 self.welcome = self._shortcmd('mode reader')
982 except NNTPPermanentError:
983 # Error 5xx, probably 'not implemented'
984 pass
985 except NNTPTemporaryError as e:
986 if e.response.startswith('480'):
987 # Need authorization before 'mode reader'
988 self.readermode_afterauth = True
989 else:
990 raise
991
992 if _have_ssl:
993 def starttls(self, context=None):
994 """Process a STARTTLS command. Arguments:
995 - context: SSL context to use for the encrypted connection
996 """
997 # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if
998 # a TLS session already exists.
999 if self.tls_on:
1000 raise ValueError("TLS is already enabled.")
1001 if self.authenticated:
1002 raise ValueError("TLS cannot be started after authentication.")
1003 resp = self._shortcmd('STARTTLS')
1004 if resp.startswith('382'):
1005 self.file.close()
1006 self.sock = _encrypt_on(self.sock, context, self.host)
1007 self.file = self.sock.makefile("rwb")
1008 self.tls_on = True
1009 # Capabilities may change after TLS starts up, so ask for them
1010 # again.
1011 self._caps = None
1012 self.getcapabilities()
1013 else:
1014 raise NNTPError("TLS failed to start.")
1015
1016
1017 if _have_ssl:
1018 class ESC[4;38;5;81mNNTP_SSL(ESC[4;38;5;149mNNTP):
1019
1020 def __init__(self, host, port=NNTP_SSL_PORT,
1021 user=None, password=None, ssl_context=None,
1022 readermode=None, usenetrc=False,
1023 timeout=_GLOBAL_DEFAULT_TIMEOUT):
1024 """This works identically to NNTP.__init__, except for the change
1025 in default port and the `ssl_context` argument for SSL connections.
1026 """
1027 self.ssl_context = ssl_context
1028 super().__init__(host, port, user, password, readermode,
1029 usenetrc, timeout)
1030
1031 def _create_socket(self, timeout):
1032 sock = super()._create_socket(timeout)
1033 try:
1034 sock = _encrypt_on(sock, self.ssl_context, self.host)
1035 except:
1036 sock.close()
1037 raise
1038 else:
1039 return sock
1040
1041 __all__.append("NNTP_SSL")
1042
1043
1044 # Test retrieval when run as a script.
1045 if __name__ == '__main__':
1046 import argparse
1047
1048 parser = argparse.ArgumentParser(description="""\
1049 nntplib built-in demo - display the latest articles in a newsgroup""")
1050 parser.add_argument('-g', '--group', default='gmane.comp.python.general',
1051 help='group to fetch messages from (default: %(default)s)')
1052 parser.add_argument('-s', '--server', default='news.gmane.io',
1053 help='NNTP server hostname (default: %(default)s)')
1054 parser.add_argument('-p', '--port', default=-1, type=int,
1055 help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT))
1056 parser.add_argument('-n', '--nb-articles', default=10, type=int,
1057 help='number of articles to fetch (default: %(default)s)')
1058 parser.add_argument('-S', '--ssl', action='store_true', default=False,
1059 help='use NNTP over SSL')
1060 args = parser.parse_args()
1061
1062 port = args.port
1063 if not args.ssl:
1064 if port == -1:
1065 port = NNTP_PORT
1066 s = NNTP(host=args.server, port=port)
1067 else:
1068 if port == -1:
1069 port = NNTP_SSL_PORT
1070 s = NNTP_SSL(host=args.server, port=port)
1071
1072 caps = s.getcapabilities()
1073 if 'STARTTLS' in caps:
1074 s.starttls()
1075 resp, count, first, last, name = s.group(args.group)
1076 print('Group', name, 'has', count, 'articles, range', first, 'to', last)
1077
1078 def cut(s, lim):
1079 if len(s) > lim:
1080 s = s[:lim - 4] + "..."
1081 return s
1082
1083 first = str(int(last) - args.nb_articles + 1)
1084 resp, overviews = s.xover(first, last)
1085 for artnum, over in overviews:
1086 author = decode_header(over['from']).split('<', 1)[0]
1087 subject = decode_header(over['subject'])
1088 lines = int(over[':lines'])
1089 print("{:7} {:20} {:42} ({})".format(
1090 artnum, cut(author, 20), cut(subject, 42), lines)
1091 )
1092
1093 s.quit()