1 # -*- test-case-name: twisted.mail.test.test_pop3 -*-
3 # Copyright (c) Twisted Matrix Laboratories.
4 # See LICENSE for details.
8 Post-office Protocol version 3
10 @author: Glyph Lefkowitz
18 from zope.interface import implements, Interface
20 from twisted.mail import smtp
21 from twisted.protocols import basic
22 from twisted.protocols import policies
23 from twisted.internet import task
24 from twisted.internet import defer
25 from twisted.internet import interfaces
26 from twisted.python import log
27 from twisted.python.hashlib import md5
29 from twisted import cred
30 import twisted.cred.error
31 import twisted.cred.credentials
36 class APOPCredentials:
37 implements(cred.credentials.IUsernamePassword)
39 def __init__(self, magic, username, digest):
41 self.username = username
44 def checkPassword(self, password):
45 seed = self.magic + password
46 myDigest = md5(seed).hexdigest()
47 return myDigest == self.digest
50 class _HeadersPlusNLines:
51 def __init__(self, f, n):
59 def read(self, bytes):
62 data = self.f.read(bytes)
66 df, sz = data.find('\r\n\r\n'), 4
68 df, sz = data.find('\n\n'), 2
77 if self.linecount > 0:
78 dsplit = (self.buf+data).split('\n')
80 for ln in dsplit[:-1]:
81 if self.linecount > self.n:
92 class _POP3MessageDeleted(Exception):
94 Internal control-flow exception. Indicates the file of a deleted message
99 class POP3Error(Exception):
104 class _IteratorBuffer(object):
107 def __init__(self, write, iterable, memoryBufferSize=None):
109 Create a _IteratorBuffer.
111 @param write: A one-argument callable which will be invoked with a list
112 of strings which have been buffered.
114 @param iterable: The source of input strings as any iterable.
116 @param memoryBufferSize: The upper limit on buffered string length,
117 beyond which the buffer will be flushed to the writer.
121 self.iterator = iter(iterable)
122 if memoryBufferSize is None:
123 memoryBufferSize = 2 ** 16
124 self.memoryBufferSize = memoryBufferSize
133 v = self.iterator.next()
134 except StopIteration:
136 self.write(self.lines)
137 # Drop some references, in case they're edges in a cycle.
138 del self.iterator, self.lines, self.write
143 self.bufSize += len(v)
144 if self.bufSize > self.memoryBufferSize:
145 self.write(self.lines)
151 def iterateLineGenerator(proto, gen):
153 Hook the given protocol instance up to the given iterator with an
154 _IteratorBuffer and schedule the result to be exhausted via the protocol.
158 @rtype: L{twisted.internet.defer.Deferred}
160 coll = _IteratorBuffer(proto.transport.writeSequence, gen)
161 return proto.schedule(coll)
165 def successResponse(response):
167 Format the given object as a positive response.
169 response = str(response)
170 return '+OK %s\r\n' % (response,)
174 def formatStatResponse(msgs):
176 Format the list of message sizes appropriately for a STAT response.
178 Yields None until it finishes computing a result, then yields a str
179 instance that is suitable for use as a response to the STAT command.
180 Intended to be used with a L{twisted.internet.task.Cooperator}.
188 yield successResponse('%d %d' % (i, bytes))
192 def formatListLines(msgs):
194 Format a list of message sizes appropriately for the lines of a LIST
197 Yields str instances formatted appropriately for use as lines in the
198 response to the LIST command. Does not include the trailing '.'.
203 yield '%d %d\r\n' % (i, size)
207 def formatListResponse(msgs):
209 Format a list of message sizes appropriately for a complete LIST response.
211 Yields str instances formatted appropriately for use as a LIST command
214 yield successResponse(len(msgs))
215 for ele in formatListLines(msgs):
221 def formatUIDListLines(msgs, getUidl):
223 Format the list of message sizes appropriately for the lines of a UIDL
226 Yields str instances formatted appropriately for use as lines in the
227 response to the UIDL command. Does not include the trailing '.'.
229 for i, m in enumerate(msgs):
232 yield '%d %s\r\n' % (i + 1, uid)
236 def formatUIDListResponse(msgs, getUidl):
238 Format a list of message sizes appropriately for a complete UIDL response.
240 Yields str instances formatted appropriately for use as a UIDL command
243 yield successResponse('')
244 for ele in formatUIDListLines(msgs, getUidl):
250 class POP3(basic.LineOnlyReceiver, policies.TimeoutMixin):
252 POP3 server protocol implementation.
254 @ivar portal: A reference to the L{twisted.cred.portal.Portal} instance we
255 will authenticate through.
257 @ivar factory: A L{twisted.mail.pop3.IServerFactory} which will be used to
258 determine some extended behavior of the server.
260 @ivar timeOut: An integer which defines the minimum amount of time which
261 may elapse without receiving any traffic after which the client will be
264 @ivar schedule: A one-argument callable which should behave like
265 L{twisted.internet.task.coiterate}.
267 implements(interfaces.IProducer)
273 AUTH_CMDS = ['CAPA', 'USER', 'PASS', 'APOP', 'AUTH', 'RPOP', 'QUIT']
278 # The mailbox we're serving
281 # Set this pretty low -- POP3 clients are expected to log in, download
282 # everything, and log out.
285 # Current protocol state
291 # Cooperate and suchlike.
292 schedule = staticmethod(task.coiterate)
294 # Message index of the highest retrieved message.
297 def connectionMade(self):
298 if self.magic is None:
299 self.magic = self.generateMagic()
300 self.successResponse(self.magic)
301 self.setTimeout(self.timeOut)
302 if getattr(self.factory, 'noisy', True):
303 log.msg("New connection from " + str(self.transport.getPeer()))
306 def connectionLost(self, reason):
307 if self._onLogout is not None:
309 self._onLogout = None
310 self.setTimeout(None)
313 def generateMagic(self):
314 return smtp.messageid()
317 def successResponse(self, message=''):
318 self.transport.write(successResponse(message))
320 def failResponse(self, message=''):
321 self.sendLine('-ERR ' + str(message))
323 # def sendLine(self, line):
324 # print 'S:', repr(line)
325 # basic.LineOnlyReceiver.sendLine(self, line)
327 def lineReceived(self, line):
328 # print 'C:', repr(line)
330 getattr(self, 'state_' + self.state)(line)
332 def _unblock(self, _):
333 commands = self.blocked
335 while commands and self.blocked is None:
336 cmd, args = commands.pop(0)
337 self.processCommand(cmd, *args)
338 if self.blocked is not None:
339 self.blocked.extend(commands)
341 def state_COMMAND(self, line):
343 return self.processCommand(*line.split(' '))
344 except (ValueError, AttributeError, POP3Error, TypeError), e:
346 self.failResponse('bad protocol or server: %s: %s' % (e.__class__.__name__, e))
348 def processCommand(self, command, *args):
349 if self.blocked is not None:
350 self.blocked.append((command, args))
353 command = command.upper()
354 authCmd = command in self.AUTH_CMDS
355 if not self.mbox and not authCmd:
356 raise POP3Error("not authenticated yet: cannot do " + command)
357 f = getattr(self, 'do_' + command, None)
360 raise POP3Error("Unknown protocol command: " + command)
363 def listCapabilities(self):
374 if IServerFactory.providedBy(self.factory):
375 # Oh my god. We can't just loop over a list of these because
376 # each has spectacularly different return value semantics!
378 v = self.factory.cap_IMPLEMENTATION()
379 except NotImplementedError:
384 baseCaps.append("IMPLEMENTATION " + str(v))
387 v = self.factory.cap_EXPIRE()
388 except NotImplementedError:
395 if self.factory.perUserExpiration():
397 v = str(self.mbox.messageExpiration)
401 baseCaps.append("EXPIRE " + v)
404 v = self.factory.cap_LOGIN_DELAY()
405 except NotImplementedError:
410 if self.factory.perUserLoginDelay():
412 v = str(self.mbox.loginDelay)
416 baseCaps.append("LOGIN-DELAY " + v)
419 v = self.factory.challengers
420 except AttributeError:
425 baseCaps.append("SASL " + ' '.join(v.keys()))
429 self.successResponse("I can do the following:")
430 for cap in self.listCapabilities():
434 def do_AUTH(self, args=None):
435 if not getattr(self.factory, 'challengers', None):
436 self.failResponse("AUTH extension unsupported")
440 self.successResponse("Supported authentication methods:")
441 for a in self.factory.challengers:
442 self.sendLine(a.upper())
446 auth = self.factory.challengers.get(args.strip().upper())
447 if not self.portal or not auth:
448 self.failResponse("Unsupported SASL selected")
452 chal = self._auth.getChallenge()
454 self.sendLine('+ ' + base64.encodestring(chal).rstrip('\n'))
457 def state_AUTH(self, line):
458 self.state = "COMMAND"
460 parts = base64.decodestring(line).split(None, 1)
461 except binascii.Error:
462 self.failResponse("Invalid BASE64 encoding")
465 self.failResponse("Invalid AUTH response")
467 self._auth.username = parts[0]
468 self._auth.response = parts[1]
469 d = self.portal.login(self._auth, None, IMailbox)
470 d.addCallback(self._cbMailbox, parts[0])
471 d.addErrback(self._ebMailbox)
472 d.addErrback(self._ebUnexpected)
474 def do_APOP(self, user, digest):
475 d = defer.maybeDeferred(self.authenticateUserAPOP, user, digest)
476 d.addCallbacks(self._cbMailbox, self._ebMailbox, callbackArgs=(user,)
477 ).addErrback(self._ebUnexpected)
479 def _cbMailbox(self, (interface, avatar, logout), user):
480 if interface is not IMailbox:
481 self.failResponse('Authentication failed')
482 log.err("_cbMailbox() called with an interface other than IMailbox")
486 self._onLogout = logout
487 self.successResponse('Authentication succeeded')
488 if getattr(self.factory, 'noisy', True):
489 log.msg("Authenticated login for " + user)
491 def _ebMailbox(self, failure):
492 failure = failure.trap(cred.error.LoginDenied, cred.error.LoginFailed)
493 if issubclass(failure, cred.error.LoginDenied):
494 self.failResponse("Access denied: " + str(failure))
495 elif issubclass(failure, cred.error.LoginFailed):
496 self.failResponse('Authentication failed')
497 if getattr(self.factory, 'noisy', True):
498 log.msg("Denied login attempt from " + str(self.transport.getPeer()))
500 def _ebUnexpected(self, failure):
501 self.failResponse('Server error: ' + failure.getErrorMessage())
504 def do_USER(self, user):
506 self.successResponse('USER accepted, send PASS')
508 def do_PASS(self, password):
509 if self._userIs is None:
510 self.failResponse("USER required before PASS")
514 d = defer.maybeDeferred(self.authenticateUserPASS, user, password)
515 d.addCallbacks(self._cbMailbox, self._ebMailbox, callbackArgs=(user,)
516 ).addErrback(self._ebUnexpected)
519 def _longOperation(self, d):
520 # Turn off timeouts and block further processing until the Deferred
521 # fires, then reverse those changes.
522 timeOut = self.timeOut
523 self.setTimeout(None)
525 d.addCallback(self._unblock)
526 d.addCallback(lambda ign: self.setTimeout(timeOut))
530 def _coiterate(self, gen):
531 return self.schedule(_IteratorBuffer(self.transport.writeSequence, gen))
535 d = defer.maybeDeferred(self.mbox.listMessages)
536 def cbMessages(msgs):
537 return self._coiterate(formatStatResponse(msgs))
539 self.failResponse(err.getErrorMessage())
540 log.msg("Unexpected do_STAT failure:")
542 return self._longOperation(d.addCallbacks(cbMessages, ebMessages))
545 def do_LIST(self, i=None):
547 d = defer.maybeDeferred(self.mbox.listMessages)
548 def cbMessages(msgs):
549 return self._coiterate(formatListResponse(msgs))
551 self.failResponse(err.getErrorMessage())
552 log.msg("Unexpected do_LIST failure:")
554 return self._longOperation(d.addCallbacks(cbMessages, ebMessages))
561 self.failResponse("Invalid message-number: %r" % (i,))
563 d = defer.maybeDeferred(self.mbox.listMessages, i - 1)
565 self.successResponse('%d %d' % (i, msg))
567 errcls = err.check(ValueError, IndexError)
568 if errcls is not None:
569 if errcls is IndexError:
570 # IndexError was supported for a while, but really
571 # shouldn't be. One error condition, one exception
574 "twisted.mail.pop3.IMailbox.listMessages may not "
575 "raise IndexError for out-of-bounds message numbers: "
576 "raise ValueError instead.",
577 PendingDeprecationWarning)
578 self.failResponse("Invalid message-number: %r" % (i,))
580 self.failResponse(err.getErrorMessage())
581 log.msg("Unexpected do_LIST failure:")
583 return self._longOperation(d.addCallbacks(cbMessage, ebMessage))
586 def do_UIDL(self, i=None):
588 d = defer.maybeDeferred(self.mbox.listMessages)
589 def cbMessages(msgs):
590 return self._coiterate(formatUIDListResponse(msgs, self.mbox.getUidl))
592 self.failResponse(err.getErrorMessage())
593 log.msg("Unexpected do_UIDL failure:")
595 return self._longOperation(d.addCallbacks(cbMessages, ebMessages))
602 self.failResponse("Bad message number argument")
605 msg = self.mbox.getUidl(i - 1)
607 # XXX TODO See above comment regarding IndexError.
609 "twisted.mail.pop3.IMailbox.getUidl may not "
610 "raise IndexError for out-of-bounds message numbers: "
611 "raise ValueError instead.",
612 PendingDeprecationWarning)
613 self.failResponse("Bad message number argument")
615 self.failResponse("Bad message number argument")
617 self.successResponse(str(msg))
620 def _getMessageFile(self, i):
622 Retrieve the size and contents of a given message, as a two-tuple.
624 @param i: The number of the message to operate on. This is a base-ten
625 string representation starting at 1.
627 @return: A Deferred which fires with a two-tuple of an integer and a
635 self.failResponse("Bad message number argument")
636 return defer.succeed(None)
638 sizeDeferred = defer.maybeDeferred(self.mbox.listMessages, msg)
639 def cbMessageSize(size):
641 return defer.fail(_POP3MessageDeleted())
642 fileDeferred = defer.maybeDeferred(self.mbox.getMessage, msg)
643 fileDeferred.addCallback(lambda fObj: (size, fObj))
646 def ebMessageSomething(err):
647 errcls = err.check(_POP3MessageDeleted, ValueError, IndexError)
648 if errcls is _POP3MessageDeleted:
649 self.failResponse("message deleted")
650 elif errcls in (ValueError, IndexError):
651 if errcls is IndexError:
652 # XXX TODO See above comment regarding IndexError.
654 "twisted.mail.pop3.IMailbox.listMessages may not "
655 "raise IndexError for out-of-bounds message numbers: "
656 "raise ValueError instead.",
657 PendingDeprecationWarning)
658 self.failResponse("Bad message number argument")
660 log.msg("Unexpected _getMessageFile failure:")
664 sizeDeferred.addCallback(cbMessageSize)
665 sizeDeferred.addErrback(ebMessageSomething)
669 def _sendMessageContent(self, i, fpWrapper, successResponse):
670 d = self._getMessageFile(i)
671 def cbMessageFile(info):
673 # Some error occurred - a failure response has been sent
674 # already, just give up.
677 self._highest = max(self._highest, int(i))
680 self.successResponse(successResponse(resp))
681 s = basic.FileSender()
682 d = s.beginFileTransfer(fp, self.transport, self.transformChunk)
684 def cbFileTransfer(lastsent):
691 def ebFileTransfer(err):
692 self.transport.loseConnection()
693 log.msg("Unexpected error in _sendMessageContent:")
696 d.addCallback(cbFileTransfer)
697 d.addErrback(ebFileTransfer)
699 return self._longOperation(d.addCallback(cbMessageFile))
702 def do_TOP(self, i, size):
708 self.failResponse("Bad line count argument")
710 return self._sendMessageContent(
712 lambda fp: _HeadersPlusNLines(fp, size),
713 lambda size: "Top of message follows")
716 def do_RETR(self, i):
717 return self._sendMessageContent(
720 lambda size: "%d" % (size,))
723 def transformChunk(self, chunk):
724 return chunk.replace('\n', '\r\n').replace('\r\n.', '\r\n..')
727 def finishedFileTransfer(self, lastsent):
735 def do_DELE(self, i):
737 self.mbox.deleteMessage(i)
738 self.successResponse()
742 """Perform no operation. Return a success code"""
743 self.successResponse()
747 """Unset all deleted message flags"""
749 self.mbox.undeleteMessages()
755 self.successResponse()
760 Return the index of the highest message yet downloaded.
762 self.successResponse(self._highest)
765 def do_RPOP(self, user):
766 self.failResponse('permission denied, sucker')
772 self.successResponse()
773 self.transport.loseConnection()
776 def authenticateUserAPOP(self, user, digest):
777 """Perform authentication of an APOP login.
780 @param user: The name of the user attempting to log in.
783 @param digest: The response string with which the user replied.
786 @return: A deferred whose callback is invoked if the login is
787 successful, and whose errback will be invoked otherwise. The
788 callback will be passed a 3-tuple consisting of IMailbox,
789 an object implementing IMailbox, and a zero-argument callable
790 to be invoked when this session is terminated.
792 if self.portal is not None:
793 return self.portal.login(
794 APOPCredentials(self.magic, user, digest),
798 raise cred.error.UnauthorizedLogin()
800 def authenticateUserPASS(self, user, password):
801 """Perform authentication of a username/password login.
804 @param user: The name of the user attempting to log in.
806 @type password: C{str}
807 @param password: The password to attempt to authenticate with.
810 @return: A deferred whose callback is invoked if the login is
811 successful, and whose errback will be invoked otherwise. The
812 callback will be passed a 3-tuple consisting of IMailbox,
813 an object implementing IMailbox, and a zero-argument callable
814 to be invoked when this session is terminated.
816 if self.portal is not None:
817 return self.portal.login(
818 cred.credentials.UsernamePassword(user, password),
822 raise cred.error.UnauthorizedLogin()
825 class IServerFactory(Interface):
826 """Interface for querying additional parameters of this POP3 server.
828 Any cap_* method may raise NotImplementedError if the particular
829 capability is not supported. If cap_EXPIRE() does not raise
830 NotImplementedError, perUserExpiration() must be implemented, otherwise
831 they are optional. If cap_LOGIN_DELAY() is implemented,
832 perUserLoginDelay() must be implemented, otherwise they are optional.
834 @ivar challengers: A dictionary mapping challenger names to classes
835 implementing C{IUsernameHashedPassword}.
838 def cap_IMPLEMENTATION():
839 """Return a string describing this POP3 server implementation."""
842 """Return the minimum number of days messages are retained."""
844 def perUserExpiration():
845 """Indicate whether message expiration is per-user.
847 @return: True if it is, false otherwise.
850 def cap_LOGIN_DELAY():
851 """Return the minimum number of seconds between client logins."""
853 def perUserLoginDelay():
854 """Indicate whether the login delay period is per-user.
856 @return: True if it is, false otherwise.
859 class IMailbox(Interface):
861 @type loginDelay: C{int}
862 @ivar loginDelay: The number of seconds between allowed logins for the
863 user associated with this mailbox. None
865 @type messageExpiration: C{int}
866 @ivar messageExpiration: The number of days messages in this mailbox will
867 remain on the server before being deleted.
870 def listMessages(index=None):
871 """Retrieve the size of one or more messages.
873 @type index: C{int} or C{None}
874 @param index: The number of the message for which to retrieve the
875 size (starting at 0), or None to retrieve the size of all messages.
877 @rtype: C{int} or any iterable of C{int} or a L{Deferred} which fires
880 @return: The number of octets in the specified message, or an iterable
881 of integers representing the number of octets in all the messages. Any
882 value which would have referred to a deleted message should be set to 0.
884 @raise ValueError: if C{index} is greater than the index of any message
888 def getMessage(index):
889 """Retrieve a file-like object for a particular message.
892 @param index: The number of the message to retrieve
894 @rtype: A file-like object
895 @return: A file containing the message data with lines delimited by
900 """Get a unique identifier for a particular message.
903 @param index: The number of the message for which to retrieve a UIDL
906 @return: A string of printable characters uniquely identifying for all
907 time the specified message.
909 @raise ValueError: if C{index} is greater than the index of any message
913 def deleteMessage(index):
914 """Delete a particular message.
916 This must not change the number of messages in this mailbox. Further
917 requests for the size of deleted messages should return 0. Further
918 requests for the message itself may raise an exception.
921 @param index: The number of the message to delete.
924 def undeleteMessages():
926 Undelete any messages which have been marked for deletion since the
927 most recent L{sync} call.
929 Any message which can be undeleted should be returned to its
930 original position in the message sequence and retain its original
935 """Perform checkpointing.
937 This method will be called to indicate the mailbox should attempt to
938 clean up any remaining deleted messages.
946 def listMessages(self, i=None):
948 def getMessage(self, i):
950 def getUidl(self, i):
952 def deleteMessage(self, i):
954 def undeleteMessages(self):
960 NONE, SHORT, FIRST_LONG, LONG = range(4)
965 NEXT[FIRST_LONG] = LONG
968 class POP3Client(basic.LineOnlyReceiver):
973 welcomeRe = re.compile('<(.*)>')
977 warnings.warn("twisted.mail.pop3.POP3Client is deprecated, "
978 "please use twisted.mail.pop3.AdvancedPOP3Client "
979 "instead.", DeprecationWarning,
982 def sendShort(self, command, params=None):
983 if params is not None:
984 self.sendLine('%s %s' % (command, params))
986 self.sendLine(command)
987 self.command = command
990 def sendLong(self, command, params):
992 self.sendLine('%s %s' % (command, params))
994 self.sendLine(command)
995 self.command = command
996 self.mode = FIRST_LONG
998 def handle_default(self, line):
999 if line[:-4] == '-ERR':
1002 def handle_WELCOME(self, line):
1003 code, data = line.split(' ', 1)
1005 self.transport.loseConnection()
1007 m = self.welcomeRe.match(line)
1009 self.welcomeCode = m.group(1)
1011 def _dispatch(self, command, default, *args):
1013 method = getattr(self, 'handle_'+command, default)
1014 if method is not None:
1019 def lineReceived(self, line):
1020 if self.mode == SHORT or self.mode == FIRST_LONG:
1021 self.mode = NEXT[self.mode]
1022 self._dispatch(self.command, self.handle_default, line)
1023 elif self.mode == LONG:
1025 self.mode = NEXT[self.mode]
1026 self._dispatch(self.command+'_end', None)
1030 self._dispatch(self.command+"_continue", None, line)
1032 def apopAuthenticate(self, user, password, magic):
1033 digest = md5(magic + password).hexdigest()
1034 self.apop(user, digest)
1036 def apop(self, user, digest):
1037 self.sendLong('APOP', ' '.join((user, digest)))
1039 self.sendLong('RETR', i)
1041 self.sendShort('DELE', i)
1042 def list(self, i=''):
1043 self.sendLong('LIST', i)
1044 def uidl(self, i=''):
1045 self.sendLong('UIDL', i)
1046 def user(self, name):
1047 self.sendShort('USER', name)
1048 def pass_(self, pass_):
1049 self.sendShort('PASS', pass_)
1051 self.sendShort('QUIT')
1053 from twisted.mail.pop3client import POP3Client as AdvancedPOP3Client
1054 from twisted.mail.pop3client import POP3ClientError
1055 from twisted.mail.pop3client import InsecureAuthenticationDisallowed
1056 from twisted.mail.pop3client import ServerErrorResponse
1057 from twisted.mail.pop3client import LineTooLong
1061 'IMailbox', 'IServerFactory',
1064 'POP3Error', 'POP3ClientError', 'InsecureAuthenticationDisallowed',
1065 'ServerErrorResponse', 'LineTooLong',
1068 'POP3', 'POP3Client', 'AdvancedPOP3Client',
1071 'APOPCredentials', 'Mailbox']