1 # -*- test-case-name: twisted.web.test.test_http -*-
2 # Copyright (c) Twisted Matrix Laboratories.
3 # See LICENSE for details.
6 HyperText Transfer Protocol implementation.
8 This is the basic server-side protocol implementation used by the Twisted
9 Web server. It can parse HTTP 1.0 requests and supports many HTTP 1.1
10 features as well. Additionally, some functionality implemented here is
11 also useful for HTTP clients (such as the chunked encoding parser).
15 from cStringIO import StringIO
17 import base64, binascii
25 from urlparse import urlparse as _urlparse
27 from zope.interface import implements
30 from twisted.internet import interfaces, reactor, protocol, address
31 from twisted.internet.defer import Deferred
32 from twisted.protocols import policies, basic
33 from twisted.python import log
34 from urllib import unquote
36 from twisted.web.http_headers import _DictHeaders, Headers
38 protocol_version = "HTTP/1.1"
46 NON_AUTHORITATIVE_INFORMATION = 203
53 MOVED_PERMANENTLY = 301
58 TEMPORARY_REDIRECT = 307
62 PAYMENT_REQUIRED = 402
67 PROXY_AUTH_REQUIRED = 407
72 PRECONDITION_FAILED = 412
73 REQUEST_ENTITY_TOO_LARGE = 413
74 REQUEST_URI_TOO_LONG = 414
75 UNSUPPORTED_MEDIA_TYPE = 415
76 REQUESTED_RANGE_NOT_SATISFIABLE = 416
77 EXPECTATION_FAILED = 417
79 INTERNAL_SERVER_ERROR = 500
82 SERVICE_UNAVAILABLE = 503
84 HTTP_VERSION_NOT_SUPPORTED = 505
85 INSUFFICIENT_STORAGE_SPACE = 507
90 _CONTINUE: "Continue",
91 SWITCHING: "Switching Protocols",
97 NON_AUTHORITATIVE_INFORMATION: "Non-Authoritative Information",
98 NO_CONTENT: "No Content",
99 RESET_CONTENT: "Reset Content.",
100 PARTIAL_CONTENT: "Partial Content",
101 MULTI_STATUS: "Multi-Status",
104 MULTIPLE_CHOICE: "Multiple Choices",
105 MOVED_PERMANENTLY: "Moved Permanently",
107 SEE_OTHER: "See Other",
108 NOT_MODIFIED: "Not Modified",
109 USE_PROXY: "Use Proxy",
111 TEMPORARY_REDIRECT: "Temporary Redirect",
114 BAD_REQUEST: "Bad Request",
115 UNAUTHORIZED: "Unauthorized",
116 PAYMENT_REQUIRED: "Payment Required",
117 FORBIDDEN: "Forbidden",
118 NOT_FOUND: "Not Found",
119 NOT_ALLOWED: "Method Not Allowed",
120 NOT_ACCEPTABLE: "Not Acceptable",
121 PROXY_AUTH_REQUIRED: "Proxy Authentication Required",
122 REQUEST_TIMEOUT: "Request Time-out",
123 CONFLICT: "Conflict",
125 LENGTH_REQUIRED: "Length Required",
126 PRECONDITION_FAILED: "Precondition Failed",
127 REQUEST_ENTITY_TOO_LARGE: "Request Entity Too Large",
128 REQUEST_URI_TOO_LONG: "Request-URI Too Long",
129 UNSUPPORTED_MEDIA_TYPE: "Unsupported Media Type",
130 REQUESTED_RANGE_NOT_SATISFIABLE: "Requested Range not satisfiable",
131 EXPECTATION_FAILED: "Expectation Failed",
134 INTERNAL_SERVER_ERROR: "Internal Server Error",
135 NOT_IMPLEMENTED: "Not Implemented",
136 BAD_GATEWAY: "Bad Gateway",
137 SERVICE_UNAVAILABLE: "Service Unavailable",
138 GATEWAY_TIMEOUT: "Gateway Time-out",
139 HTTP_VERSION_NOT_SUPPORTED: "HTTP Version not supported",
140 INSUFFICIENT_STORAGE_SPACE: "Insufficient Storage Space",
141 NOT_EXTENDED: "Not Extended"
144 CACHED = """Magic constant returned by http.Request methods to set cache
145 validation headers when the request is conditional and the value fails
148 # backwards compatability
149 responses = RESPONSES
152 # datetime parsing and formatting
153 weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
155 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
156 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
157 weekdayname_lower = [name.lower() for name in weekdayname]
158 monthname_lower = [name and name.lower() for name in monthname]
162 Parse an URL into six components.
164 This is similar to L{urlparse.urlparse}, but rejects C{unicode} input
165 and always produces C{str} output.
169 @raise TypeError: The given url was a C{unicode} string instead of a
172 @rtype: six-tuple of str
173 @return: The scheme, net location, path, params, query string, and fragment
176 if isinstance(url, unicode):
177 raise TypeError("url must be str, not unicode")
178 scheme, netloc, path, params, query, fragment = _urlparse(url)
179 if isinstance(scheme, unicode):
180 scheme = scheme.encode('ascii')
181 netloc = netloc.encode('ascii')
182 path = path.encode('ascii')
183 query = query.encode('ascii')
184 fragment = fragment.encode('ascii')
185 return scheme, netloc, path, params, query, fragment
188 def parse_qs(qs, keep_blank_values=0, strict_parsing=0, unquote=unquote):
190 like cgi.parse_qs, only with custom unquote function
193 items = [s2 for s1 in qs.split("&") for s2 in s1.split(";")]
196 k, v = item.split("=", 1)
201 if v or keep_blank_values:
202 k = unquote(k.replace("+", " "))
203 v = unquote(v.replace("+", " "))
210 def datetimeToString(msSinceEpoch=None):
212 Convert seconds since epoch to HTTP datetime string.
214 if msSinceEpoch == None:
215 msSinceEpoch = time.time()
216 year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
217 s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
219 day, monthname[month], year,
223 def datetimeToLogString(msSinceEpoch=None):
225 Convert seconds since epoch to log datetime string.
227 if msSinceEpoch == None:
228 msSinceEpoch = time.time()
229 year, month, day, hh, mm, ss, wd, y, z = time.gmtime(msSinceEpoch)
230 s = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % (
231 day, monthname[month], year,
235 def timegm(year, month, day, hour, minute, second):
237 Convert time tuple in GMT to seconds since epoch, GMT
241 raise ValueError("Years prior to %d not supported" % (EPOCH,))
242 assert 1 <= month <= 12
243 days = 365*(year-EPOCH) + calendar.leapdays(EPOCH, year)
244 for i in range(1, month):
245 days = days + calendar.mdays[i]
246 if month > 2 and calendar.isleap(year):
248 days = days + day - 1
249 hours = days*24 + hour
250 minutes = hours*60 + minute
251 seconds = minutes*60 + second
254 def stringToDatetime(dateString):
256 Convert an HTTP date string (one of three formats) to seconds since epoch.
258 parts = dateString.split()
260 if not parts[0][0:3].lower() in weekdayname_lower:
261 # Weekday is stupid. Might have been omitted.
263 return stringToDatetime("Sun, "+dateString)
269 if (partlen == 5 or partlen == 6) and parts[1].isdigit():
270 # 1st date format: Sun, 06 Nov 1994 08:49:37 GMT
271 # (Note: "GMT" is literal, not a variable timezone)
272 # (also handles without "GMT")
273 # This is the normal format
278 elif (partlen == 3 or partlen == 4) and parts[1].find('-') != -1:
279 # 2nd date format: Sunday, 06-Nov-94 08:49:37 GMT
280 # (Note: "GMT" is literal, not a variable timezone)
281 # (also handles without without "GMT")
282 # Two digit year, yucko.
283 day, month, year = parts[1].split('-')
290 elif len(parts) == 5:
291 # 3rd date format: Sun Nov 6 08:49:37 1994
292 # ANSI C asctime() format.
298 raise ValueError("Unknown datetime format %r" % dateString)
301 month = int(monthname_lower.index(month.lower()))
303 hour, min, sec = map(int, time.split(':'))
304 return int(timegm(year, month, day, hour, min, sec))
308 Convert string to a chunk.
310 @returns: a tuple of strings representing the chunked encoding of data
312 return ("%x\r\n" % len(data), data, "\r\n")
316 Convert chunk to string.
318 @returns: tuple (result, remaining), may raise ValueError.
320 prefix, rest = data.split('\r\n', 1)
321 length = int(prefix, 16)
323 raise ValueError("Chunk length must be >= 0, not %d" % (length,))
324 if not rest[length:length + 2] == '\r\n':
325 raise ValueError, "chunk must end with CRLF"
326 return rest[:length], rest[length + 2:]
329 def parseContentRange(header):
331 Parse a content-range header into (start, end, realLength).
333 realLength might be None if real length is not known ('*').
335 kind, other = header.strip().split()
336 if kind.lower() != "bytes":
337 raise ValueError, "a range of type %r is not supported"
338 startend, realLength = other.split("/")
339 start, end = map(int, startend.split("-"))
340 if realLength == "*":
343 realLength = int(realLength)
344 return (start, end, realLength)
348 class StringTransport:
350 I am a StringIO wrapper that conforms for the transport API. I support
351 the `writeSequence' method.
355 def writeSequence(self, seq):
356 self.s.write(''.join(seq))
357 def __getattr__(self, attr):
358 return getattr(self.__dict__['s'], attr)
361 class HTTPClient(basic.LineReceiver):
363 A client for HTTP 1.0.
366 You probably want to send a 'Host' header with the name of the site you're
367 connecting to, in order to not break name based virtual hosting.
369 @ivar length: The length of the request body in bytes.
372 @ivar firstLine: Are we waiting for the first header line?
373 @type firstLine: C{bool}
375 @ivar __buffer: The buffer that stores the response to the HTTP request.
376 @type __buffer: A C{StringIO} object.
378 @ivar _header: Part or all of an HTTP request header.
379 @type _header: C{str}
386 def sendCommand(self, command, path):
387 self.transport.write('%s %s HTTP/1.0\r\n' % (command, path))
389 def sendHeader(self, name, value):
390 self.transport.write('%s: %s\r\n' % (name, value))
392 def endHeaders(self):
393 self.transport.write('\r\n')
396 def extractHeader(self, header):
398 Given a complete HTTP header, extract the field name and value and
401 @param header: a complete HTTP request header of the form
405 key, val = header.split(':', 1)
407 self.handleHeader(key, val)
408 if key.lower() == 'content-length':
409 self.length = int(val)
412 def lineReceived(self, line):
414 Parse the status line and headers for an HTTP request.
416 @param line: Part of an HTTP request header. Request bodies are parsed
417 in L{rawDataReceived}.
421 self.firstLine = False
422 l = line.split(None, 2)
428 # sometimes there is no message
430 self.handleStatus(version, status, message)
433 if self._header != "":
434 # Only extract headers if there are any
435 self.extractHeader(self._header)
436 self.__buffer = StringIO()
437 self.handleEndHeaders()
441 if line.startswith('\t') or line.startswith(' '):
442 # This line is part of a multiline header. According to RFC 822, in
443 # "unfolding" multiline headers you do not strip the leading
444 # whitespace on the continuing line.
445 self._header = self._header + line
447 # This line starts a new header, so process the previous one.
448 self.extractHeader(self._header)
454 def connectionLost(self, reason):
455 self.handleResponseEnd()
457 def handleResponseEnd(self):
459 The response has been completely received.
461 This callback may be invoked more than once per request.
463 if self.__buffer is not None:
464 b = self.__buffer.getvalue()
466 self.handleResponse(b)
468 def handleResponsePart(self, data):
469 self.__buffer.write(data)
471 def connectionMade(self):
474 def handleStatus(self, version, status, message):
476 Called when the status-line is received.
478 @param version: e.g. 'HTTP/1.0'
479 @param status: e.g. '200'
481 @param message: e.g. 'OK'
484 def handleHeader(self, key, val):
486 Called every time a header is received.
489 def handleEndHeaders(self):
491 Called when all headers have been received.
495 def rawDataReceived(self, data):
496 if self.length is not None:
497 data, rest = data[:self.length], data[self.length:]
498 self.length -= len(data)
501 self.handleResponsePart(data)
503 self.handleResponseEnd()
504 self.setLineMode(rest)
508 # response codes that must have empty bodies
509 NO_BODY_CODES = (204, 304)
515 Subclasses should override the process() method to determine how
516 the request will be processed.
518 @ivar method: The HTTP method that was used.
519 @ivar uri: The full URI that was requested (includes arguments).
520 @ivar path: The path only (arguments not included).
521 @ivar args: All of the arguments, including URL and POST arguments.
522 @type args: A mapping of strings (the argument names) to lists of values.
523 i.e., ?foo=bar&foo=baz&quux=spam results in
524 {'foo': ['bar', 'baz'], 'quux': ['spam']}.
526 @type requestHeaders: L{http_headers.Headers}
527 @ivar requestHeaders: All received HTTP request headers.
529 @ivar received_headers: Backwards-compatibility access to
530 C{requestHeaders}. Use C{requestHeaders} instead. C{received_headers}
531 behaves mostly like a C{dict} and does not provide access to all header
534 @type responseHeaders: L{http_headers.Headers}
535 @ivar responseHeaders: All HTTP response headers to be sent.
537 @ivar headers: Backwards-compatibility access to C{responseHeaders}. Use
538 C{responseHeaders} instead. C{headers} behaves mostly like a C{dict}
539 and does not provide access to all header values nor does it allow
540 multiple values for one header to be set.
542 @ivar notifications: A C{list} of L{Deferred}s which are waiting for
543 notification that the response to this request has been finished
544 (successfully or with an error). Don't use this attribute directly,
545 instead use the L{Request.notifyFinish} method.
547 @ivar _disconnected: A flag which is C{False} until the connection over
548 which this request was received is closed and which is C{True} after
550 @type _disconnected: C{bool}
552 implements(interfaces.IConsumer)
557 code_message = RESPONSES[OK]
558 method = "(no method yet)"
559 clientproto = "(no clientproto yet)"
563 sentLength = 0 # content-length of response, or total bytes sent via chunking
570 _disconnected = False
572 def __init__(self, channel, queued):
574 @param channel: the channel we're connected to.
575 @param queued: are we in the request queue, or can we start writing to
578 self.notifications = []
579 self.channel = channel
581 self.requestHeaders = Headers()
582 self.received_cookies = {}
583 self.responseHeaders = Headers()
584 self.cookies = [] # outgoing cookies
587 self.transport = StringTransport()
589 self.transport = self.channel.transport
592 def __setattr__(self, name, value):
594 Support assignment of C{dict} instances to C{received_headers} for
595 backwards-compatibility.
597 if name == 'received_headers':
598 # A property would be nice, but Request is classic.
599 self.requestHeaders = headers = Headers()
600 for k, v in value.iteritems():
601 headers.setRawHeaders(k, [v])
602 elif name == 'requestHeaders':
603 self.__dict__[name] = value
604 self.__dict__['received_headers'] = _DictHeaders(value)
605 elif name == 'headers':
606 self.responseHeaders = headers = Headers()
607 for k, v in value.iteritems():
608 headers.setRawHeaders(k, [v])
609 elif name == 'responseHeaders':
610 self.__dict__[name] = value
611 self.__dict__['headers'] = _DictHeaders(value)
613 self.__dict__[name] = value
618 Called when have finished responding and are no longer queued.
621 log.err(RuntimeError("Producer was not unregistered for %s" % self.uri))
622 self.unregisterProducer()
623 self.channel.requestDone(self)
628 # win32 suckiness, no idea why it does this
631 for d in self.notifications:
633 self.notifications = []
635 # methods for channel - end users should not use these
637 def noLongerQueued(self):
639 Notify the object that it is no longer queued.
641 We start writing whatever data we have to the transport, etc.
643 This method is not intended for users.
646 raise RuntimeError, "noLongerQueued() got called unnecessarily."
650 # set transport to real one and send any buffer data
651 data = self.transport.getvalue()
652 self.transport = self.channel.transport
654 self.transport.write(data)
656 # if we have producer, register it with transport
657 if (self.producer is not None) and not self.finished:
658 self.transport.registerProducer(self.producer, self.streamingProducer)
660 # if we're finished, clean up
664 def gotLength(self, length):
666 Called when HTTP channel got length of content in this request.
668 This method is not intended for users.
670 @param length: The length of the request body, as indicated by the
671 request headers. C{None} if the request headers do not indicate a
674 if length is not None and length < 100000:
675 self.content = StringIO()
677 self.content = tempfile.TemporaryFile()
680 def parseCookies(self):
682 Parse cookie headers.
684 This method is not intended for users.
686 cookieheaders = self.requestHeaders.getRawHeaders("cookie")
688 if cookieheaders is None:
691 for cookietxt in cookieheaders:
693 for cook in cookietxt.split(';'):
696 k, v = cook.split('=', 1)
697 self.received_cookies[k] = v
702 def handleContentChunk(self, data):
704 Write a chunk of data.
706 This method is not intended for users.
708 self.content.write(data)
711 def requestReceived(self, command, path, version):
713 Called by channel when all data has been received.
715 This method is not intended for users.
717 @type command: C{str}
718 @param command: The HTTP verb of this request. This has the case
719 supplied by the client (eg, it maybe "get" rather than "GET").
722 @param path: The URI of this request.
724 @type version: C{str}
725 @param version: The HTTP version of this request.
727 self.content.seek(0,0)
731 self.method, self.uri = command, path
732 self.clientproto = version
733 x = self.uri.split('?', 1)
738 self.path, argstring = x
739 self.args = parse_qs(argstring, 1)
741 # cache the client and server information, we'll need this later to be
742 # serialized and sent with the request so CGIs will work remotely
743 self.client = self.channel.transport.getPeer()
744 self.host = self.channel.transport.getHost()
746 # Argument processing
748 ctype = self.requestHeaders.getRawHeaders('content-type')
749 if ctype is not None:
752 if self.method == "POST" and ctype:
753 mfd = 'multipart/form-data'
754 key, pdict = cgi.parse_header(ctype)
755 if key == 'application/x-www-form-urlencoded':
756 args.update(parse_qs(self.content.read(), 1))
759 args.update(cgi.parse_multipart(self.content, pdict))
761 if e.args[0] == 'content-disposition':
762 # Parse_multipart can't cope with missing
763 # content-dispostion headers in multipart/form-data
764 # parts, so we catch the exception and tell the client
765 # it was a bad request.
766 self.channel.transport.write(
767 "HTTP/1.1 400 Bad Request\r\n\r\n")
768 self.channel.transport.loseConnection()
771 self.content.seek(0, 0)
777 return '<%s %s %s>'% (self.method, self.uri, self.clientproto)
781 Override in subclasses.
783 This method is not intended for users.
790 def registerProducer(self, producer, streaming):
795 raise ValueError, "registering producer %s before previous one (%s) was unregistered" % (producer, self.producer)
797 self.streamingProducer = streaming
798 self.producer = producer
802 producer.pauseProducing()
804 self.transport.registerProducer(producer, streaming)
806 def unregisterProducer(self):
808 Unregister the producer.
811 self.transport.unregisterProducer()
814 # private http response methods
816 def _sendError(self, code, resp=''):
817 self.transport.write('%s %s %s\r\n\r\n' % (self.clientproto, code, resp))
820 # The following is the public interface that people should be
822 def getHeader(self, key):
824 Get an HTTP request header.
827 @param key: The name of the header to get the value of.
829 @rtype: C{str} or C{NoneType}
830 @return: The value of the specified header, or C{None} if that header
831 was not present in the request.
833 value = self.requestHeaders.getRawHeaders(key)
834 if value is not None:
838 def getCookie(self, key):
840 Get a cookie that was sent from the network.
842 return self.received_cookies.get(key)
845 def notifyFinish(self):
847 Notify when the response to this request has finished.
851 @return: A L{Deferred} which will be triggered when the request is
852 finished -- with a C{None} value if the request finishes
853 successfully or with an error if the request is interrupted by an
854 error (for example, the client closing the connection prematurely).
856 self.notifications.append(Deferred())
857 return self.notifications[-1]
862 Indicate that all response data has been written to this L{Request}.
864 if self._disconnected:
866 "Request.finish called on a request after its connection was lost; "
867 "use Request.notifyFinish to keep track of this.")
869 warnings.warn("Warning! request.finish called twice.", stacklevel=2)
872 if not self.startedWriting:
877 # write last chunk and closing CRLF
878 self.transport.write("0\r\n\r\n")
881 if hasattr(self.channel, "factory"):
882 self.channel.factory.log(self)
889 def write(self, data):
891 Write some data as a result of an HTTP request. The first
892 time this is called, it writes out response data.
895 @param data: Some bytes to be sent as part of the response body.
898 raise RuntimeError('Request.write called on a request after '
899 'Request.finish was called.')
900 if not self.startedWriting:
901 self.startedWriting = 1
902 version = self.clientproto
904 l.append('%s %s %s\r\n' % (version, self.code,
906 # if we don't have a content length, we send data in
907 # chunked mode, so that we can support pipelining in
908 # persistent connections.
909 if ((version == "HTTP/1.1") and
910 (self.responseHeaders.getRawHeaders('content-length') is None) and
911 self.method != "HEAD" and self.code not in NO_BODY_CODES):
912 l.append("%s: %s\r\n" % ('Transfer-Encoding', 'chunked'))
915 if self.lastModified is not None:
916 if self.responseHeaders.hasHeader('last-modified'):
917 log.msg("Warning: last-modified specified both in"
918 " header list and lastModified attribute.")
920 self.responseHeaders.setRawHeaders(
922 [datetimeToString(self.lastModified)])
924 if self.etag is not None:
925 self.responseHeaders.setRawHeaders('ETag', [self.etag])
927 for name, values in self.responseHeaders.getAllRawHeaders():
929 l.append("%s: %s\r\n" % (name, value))
931 for cookie in self.cookies:
932 l.append('%s: %s\r\n' % ("Set-Cookie", cookie))
936 self.transport.writeSequence(l)
938 # if this is a "HEAD" request, we shouldn't return any data
939 if self.method == "HEAD":
940 self.write = lambda data: None
943 # for certain result codes, we should never return any data
944 if self.code in NO_BODY_CODES:
945 self.write = lambda data: None
948 self.sentLength = self.sentLength + len(data)
951 self.transport.writeSequence(toChunk(data))
953 self.transport.write(data)
955 def addCookie(self, k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
957 Set an outgoing HTTP cookie.
959 In general, you should consider using sessions instead of cookies, see
960 L{twisted.web.server.Request.getSession} and the
961 L{twisted.web.server.Session} class for details.
963 cookie = '%s=%s' % (k, v)
964 if expires is not None:
965 cookie = cookie +"; Expires=%s" % expires
966 if domain is not None:
967 cookie = cookie +"; Domain=%s" % domain
969 cookie = cookie +"; Path=%s" % path
970 if max_age is not None:
971 cookie = cookie +"; Max-Age=%s" % max_age
972 if comment is not None:
973 cookie = cookie +"; Comment=%s" % comment
975 cookie = cookie +"; Secure"
976 self.cookies.append(cookie)
978 def setResponseCode(self, code, message=None):
980 Set the HTTP response code.
982 if not isinstance(code, (int, long)):
983 raise TypeError("HTTP response code must be int or long")
986 self.code_message = message
988 self.code_message = RESPONSES.get(code, "Unknown Status")
991 def setHeader(self, name, value):
993 Set an HTTP response header. Overrides any previously set values for
997 @param name: The name of the header for which to set the value.
1000 @param value: The value to set for the named header.
1002 self.responseHeaders.setRawHeaders(name, [value])
1005 def redirect(self, url):
1007 Utility function that does a redirect.
1009 The request should have finish() called after this.
1011 self.setResponseCode(FOUND)
1012 self.setHeader("location", url)
1015 def setLastModified(self, when):
1017 Set the C{Last-Modified} time for the response to this request.
1019 If I am called more than once, I ignore attempts to set
1020 Last-Modified earlier, only replacing the Last-Modified time
1021 if it is to a later value.
1023 If I am a conditional request, I may modify my response code
1024 to L{NOT_MODIFIED} if appropriate for the time given.
1026 @param when: The last time the resource being returned was
1027 modified, in seconds since the epoch.
1029 @return: If I am a C{If-Modified-Since} conditional request and
1030 the time given is not newer than the condition, I return
1031 L{http.CACHED<CACHED>} to indicate that you should write no
1032 body. Otherwise, I return a false value.
1034 # time.time() may be a float, but the HTTP-date strings are
1035 # only good for whole seconds.
1036 when = long(math.ceil(when))
1037 if (not self.lastModified) or (self.lastModified < when):
1038 self.lastModified = when
1040 modifiedSince = self.getHeader('if-modified-since')
1042 firstPart = modifiedSince.split(';', 1)[0]
1044 modifiedSince = stringToDatetime(firstPart)
1047 if modifiedSince >= when:
1048 self.setResponseCode(NOT_MODIFIED)
1052 def setETag(self, etag):
1054 Set an C{entity tag} for the outgoing response.
1056 That's \"entity tag\" as in the HTTP/1.1 C{ETag} header, \"used
1057 for comparing two or more entities from the same requested
1060 If I am a conditional request, I may modify my response code
1061 to L{NOT_MODIFIED} or L{PRECONDITION_FAILED}, if appropriate
1064 @param etag: The entity tag for the resource being returned.
1066 @return: If I am a C{If-None-Match} conditional request and
1067 the tag matches one in the request, I return
1068 L{http.CACHED<CACHED>} to indicate that you should write
1069 no body. Otherwise, I return a false value.
1074 tags = self.getHeader("if-none-match")
1077 if (etag in tags) or ('*' in tags):
1078 self.setResponseCode(((self.method in ("HEAD", "GET"))
1080 or PRECONDITION_FAILED)
1085 def getAllHeaders(self):
1087 Return dictionary mapping the names of all received headers to the last
1088 value received for each.
1090 Since this method does not return all header information,
1091 C{self.requestHeaders.getAllRawHeaders()} may be preferred.
1094 for k, v in self.requestHeaders.getAllRawHeaders():
1095 headers[k.lower()] = v[-1]
1099 def getRequestHostname(self):
1101 Get the hostname that the user passed in to the request.
1103 This will either use the Host: header (if it is available) or the
1104 host we are listening on if the header is unavailable.
1106 @returns: the requested hostname
1109 # XXX This method probably has no unit tests. I changed it a ton and
1111 host = self.getHeader('host')
1113 return host.split(':', 1)[0]
1114 return self.getHost().host
1119 Get my originally requesting transport's host.
1121 Don't rely on the 'transport' attribute, since Request objects may be
1122 copied remotely. For information on this method's return value, see
1123 twisted.internet.tcp.Port.
1127 def setHost(self, host, port, ssl=0):
1129 Change the host and port the request thinks it's using.
1131 This method is useful for working with reverse HTTP proxies (e.g.
1132 both Squid and Apache's mod_proxy can do this), when the address
1133 the HTTP client is using is different than the one we're listening on.
1135 For example, Apache may be listening on https://www.example.com, and then
1136 forwarding requests to http://localhost:8080, but we don't want HTML produced
1137 by Twisted to say 'http://localhost:8080', they should say 'https://www.example.com',
1140 request.setHost('www.example.com', 443, ssl=1)
1143 @param host: The value to which to change the host header.
1146 @param ssl: A flag which, if C{True}, indicates that the request is
1147 considered secure (if C{True}, L{isSecure} will return C{True}).
1149 self._forceSSL = ssl # set first so isSecure will work
1157 hostHeader = '%s:%d' % (host, port)
1158 self.requestHeaders.setRawHeaders("host", [hostHeader])
1159 self.host = address.IPv4Address("TCP", host, port)
1162 def getClientIP(self):
1164 Return the IP address of the client who submitted this request.
1166 @returns: the client IP address
1169 if isinstance(self.client, address.IPv4Address):
1170 return self.client.host
1176 Return True if this request is using a secure transport.
1178 Normally this method returns True if this request's HTTPChannel
1179 instance is using a transport that implements ISSLTransport.
1181 This will also return True if setHost() has been called
1184 @returns: True if this request is secure
1189 transport = getattr(getattr(self, 'channel', None), 'transport', None)
1190 if interfaces.ISSLTransport(transport, None) is not None:
1194 def _authorize(self):
1195 # Authorization, (mostly) per the RFC
1197 authh = self.getHeader("Authorization")
1199 self.user = self.password = ''
1201 bas, upw = authh.split()
1202 if bas.lower() != "basic":
1204 upw = base64.decodestring(upw)
1205 self.user, self.password = upw.split(':', 1)
1206 except (binascii.Error, ValueError):
1207 self.user = self.password = ""
1210 self.user = self.password = ""
1214 Return the HTTP user sent with this request, if any.
1216 If no user was supplied, return the empty string.
1218 @returns: the HTTP user, if any
1228 def getPassword(self):
1230 Return the HTTP password sent with this request, if any.
1232 If no password was supplied, return the empty string.
1234 @returns: the HTTP password, if any
1238 return self.password
1242 return self.password
1244 def getClient(self):
1245 if self.client.type != 'TCP':
1247 host = self.client.host
1249 name, names, addresses = socket.gethostbyaddr(host)
1250 except socket.error:
1252 names.insert(0, name)
1259 def connectionLost(self, reason):
1261 There is no longer a connection for this request to respond over.
1262 Clean up anything which can't be useful anymore.
1264 self._disconnected = True
1266 if self.content is not None:
1267 self.content.close()
1268 for d in self.notifications:
1270 self.notifications = []
1274 class _DataLoss(Exception):
1276 L{_DataLoss} indicates that not all of a message body was received. This
1277 is only one of several possible exceptions which may indicate that data
1278 was lost. Because of this, it should not be checked for by
1279 specifically; any unexpected exception should be treated as having
1285 class PotentialDataLoss(Exception):
1287 L{PotentialDataLoss} may be raised by a transfer encoding decoder's
1288 C{noMoreData} method to indicate that it cannot be determined if the
1289 entire response body has been delivered. This only occurs when making
1290 requests to HTTP servers which do not set I{Content-Length} or a
1291 I{Transfer-Encoding} in the response because in this case the end of the
1292 response is indicated by the connection being closed, an event which may
1293 also be due to a transient network problem or other error.
1298 class _IdentityTransferDecoder(object):
1300 Protocol for accumulating bytes up to a specified length. This handles the
1301 case where no I{Transfer-Encoding} is specified.
1303 @ivar contentLength: Counter keeping track of how many more bytes there are
1306 @ivar dataCallback: A one-argument callable which will be invoked each
1307 time application data is received.
1309 @ivar finishCallback: A one-argument callable which will be invoked when
1310 the terminal chunk is received. It will be invoked with all bytes
1311 which were delivered to this protocol which came after the terminal
1314 def __init__(self, contentLength, dataCallback, finishCallback):
1315 self.contentLength = contentLength
1316 self.dataCallback = dataCallback
1317 self.finishCallback = finishCallback
1320 def dataReceived(self, data):
1322 Interpret the next chunk of bytes received. Either deliver them to the
1323 data callback or invoke the finish callback if enough bytes have been
1326 @raise RuntimeError: If the finish callback has already been invoked
1327 during a previous call to this methood.
1329 if self.dataCallback is None:
1331 "_IdentityTransferDecoder cannot decode data after finishing")
1333 if self.contentLength is None:
1334 self.dataCallback(data)
1335 elif len(data) < self.contentLength:
1336 self.contentLength -= len(data)
1337 self.dataCallback(data)
1339 # Make the state consistent before invoking any code belonging to
1340 # anyone else in case noMoreData ends up being called beneath this
1342 contentLength = self.contentLength
1343 dataCallback = self.dataCallback
1344 finishCallback = self.finishCallback
1345 self.dataCallback = self.finishCallback = None
1346 self.contentLength = 0
1348 dataCallback(data[:contentLength])
1349 finishCallback(data[contentLength:])
1352 def noMoreData(self):
1354 All data which will be delivered to this decoder has been. Check to
1355 make sure as much data as was expected has been received.
1357 @raise PotentialDataLoss: If the content length is unknown.
1358 @raise _DataLoss: If the content length is known and fewer than that
1359 many bytes have been delivered.
1363 finishCallback = self.finishCallback
1364 self.dataCallback = self.finishCallback = None
1365 if self.contentLength is None:
1367 raise PotentialDataLoss()
1368 elif self.contentLength != 0:
1373 class _ChunkedTransferDecoder(object):
1375 Protocol for decoding I{chunked} Transfer-Encoding, as defined by RFC 2616,
1376 section 3.6.1. This protocol can interpret the contents of a request or
1377 response body which uses the I{chunked} Transfer-Encoding. It cannot
1378 interpret any of the rest of the HTTP protocol.
1380 It may make sense for _ChunkedTransferDecoder to be an actual IProtocol
1381 implementation. Currently, the only user of this class will only ever
1382 call dataReceived on it. However, it might be an improvement if the
1383 user could connect this to a transport and deliver connection lost
1384 notification. This way, `dataCallback` becomes `self.transport.write`
1385 and perhaps `finishCallback` becomes `self.transport.loseConnection()`
1386 (although I'm not sure where the extra data goes in that case). This
1387 could also allow this object to indicate to the receiver of data that
1388 the stream was not completely received, an error case which should be
1391 @ivar dataCallback: A one-argument callable which will be invoked each
1392 time application data is received.
1394 @ivar finishCallback: A one-argument callable which will be invoked when
1395 the terminal chunk is received. It will be invoked with all bytes
1396 which were delivered to this protocol which came after the terminal
1399 @ivar length: Counter keeping track of how many more bytes in a chunk there
1402 @ivar state: One of C{'chunk-length'}, C{'trailer'}, C{'body'}, or
1403 C{'finished'}. For C{'chunk-length'}, data for the chunk length line
1404 is currently being read. For C{'trailer'}, the CR LF pair which
1405 follows each chunk is being read. For C{'body'}, the contents of a
1406 chunk are being read. For C{'finished'}, the last chunk has been
1407 completely read and no more input is valid.
1409 @ivar finish: A flag indicating that the last chunk has been started. When
1410 it finishes, the state will change to C{'finished'} and no more data
1413 state = 'chunk-length'
1416 def __init__(self, dataCallback, finishCallback):
1417 self.dataCallback = dataCallback
1418 self.finishCallback = finishCallback
1422 def dataReceived(self, data):
1424 Interpret data from a request or response body which uses the
1425 I{chunked} Transfer-Encoding.
1427 data = self._buffer + data
1430 if self.state == 'chunk-length':
1432 line, rest = data.split('\r\n', 1)
1433 parts = line.split(';')
1434 self.length = int(parts[0], 16)
1435 if self.length == 0:
1436 self.state = 'trailer'
1444 elif self.state == 'trailer':
1445 if data.startswith('\r\n'):
1448 self.state = 'finished'
1449 self.finishCallback(data)
1452 self.state = 'chunk-length'
1456 elif self.state == 'body':
1457 if len(data) >= self.length:
1458 chunk, data = data[:self.length], data[self.length:]
1459 self.dataCallback(chunk)
1460 self.state = 'trailer'
1461 elif len(data) < self.length:
1462 self.length -= len(data)
1463 self.dataCallback(data)
1465 elif self.state == 'finished':
1467 "_ChunkedTransferDecoder.dataReceived called after last "
1468 "chunk was processed")
1471 def noMoreData(self):
1473 Verify that all data has been received. If it has not been, raise
1476 if self.state != 'finished':
1478 "Chunked decoder in %r state, still expecting more data to "
1479 "get to finished state." % (self.state,))
1483 class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
1485 A receiver for HTTP requests.
1487 @ivar _transferDecoder: C{None} or an instance of
1488 L{_ChunkedTransferDecoder} if the request body uses the I{chunked}
1492 maxHeaders = 500 # max number of headers allowed per request
1500 # set in instances or subclasses
1501 requestFactory = Request
1503 _savedTimeOut = None
1504 _receivedHeaderCount = 0
1509 self._transferDecoder = None
1512 def connectionMade(self):
1513 self.setTimeout(self.timeOut)
1515 def lineReceived(self, line):
1518 if self.__first_line:
1519 # if this connection is not persistent, drop any data which
1520 # the client (illegally) sent after the last request.
1521 if not self.persistent:
1522 self.dataReceived = self.lineReceived = lambda *args: None
1525 # IE sends an extraneous empty line (\r\n) after a POST request;
1526 # eat up such a line, but only ONCE
1527 if not line and self.__first_line == 1:
1528 self.__first_line = 2
1531 # create a new Request object
1532 request = self.requestFactory(self, len(self.requests))
1533 self.requests.append(request)
1535 self.__first_line = 0
1536 parts = line.split()
1538 self.transport.write("HTTP/1.1 400 Bad Request\r\n\r\n")
1539 self.transport.loseConnection()
1541 command, request, version = parts
1542 self._command = command
1543 self._path = request
1544 self._version = version
1547 self.headerReceived(self.__header)
1549 self.allHeadersReceived()
1550 if self.length == 0:
1551 self.allContentReceived()
1554 elif line[0] in ' \t':
1555 self.__header = self.__header+'\n'+line
1558 self.headerReceived(self.__header)
1559 self.__header = line
1562 def _finishRequestBody(self, data):
1563 self.allContentReceived()
1564 self.setLineMode(data)
1567 def headerReceived(self, line):
1569 Do pre-processing (for content-length) and store this header away.
1570 Enforce the per-request header limit.
1573 @param line: A line from the header section of a request, excluding the
1576 header, data = line.split(':', 1)
1577 header = header.lower()
1579 if header == 'content-length':
1580 self.length = int(data)
1581 self._transferDecoder = _IdentityTransferDecoder(
1582 self.length, self.requests[-1].handleContentChunk, self._finishRequestBody)
1583 elif header == 'transfer-encoding' and data.lower() == 'chunked':
1585 self._transferDecoder = _ChunkedTransferDecoder(
1586 self.requests[-1].handleContentChunk, self._finishRequestBody)
1587 reqHeaders = self.requests[-1].requestHeaders
1588 values = reqHeaders.getRawHeaders(header)
1589 if values is not None:
1592 reqHeaders.setRawHeaders(header, [data])
1594 self._receivedHeaderCount += 1
1595 if self._receivedHeaderCount > self.maxHeaders:
1596 self.transport.write("HTTP/1.1 400 Bad Request\r\n\r\n")
1597 self.transport.loseConnection()
1600 def allContentReceived(self):
1601 command = self._command
1603 version = self._version
1605 # reset ALL state variables, so we don't interfere with next request
1607 self._receivedHeaderCount = 0
1608 self.__first_line = 1
1609 self._transferDecoder = None
1610 del self._command, self._path, self._version
1612 # Disable the idle timeout, in case this request takes a long
1613 # time to finish generating output.
1615 self._savedTimeOut = self.setTimeout(None)
1617 req = self.requests[-1]
1618 req.requestReceived(command, path, version)
1620 def rawDataReceived(self, data):
1622 self._transferDecoder.dataReceived(data)
1625 def allHeadersReceived(self):
1626 req = self.requests[-1]
1628 self.persistent = self.checkPersistence(req, self._version)
1629 req.gotLength(self.length)
1630 # Handle 'Expect: 100-continue' with automated 100 response code,
1631 # a simplistic implementation of RFC 2686 8.2.3:
1632 expectContinue = req.requestHeaders.getRawHeaders('expect')
1633 if (expectContinue and expectContinue[0].lower() == '100-continue' and
1634 self._version == 'HTTP/1.1'):
1635 req.transport.write("HTTP/1.1 100 Continue\r\n\r\n")
1638 def checkPersistence(self, request, version):
1640 Check if the channel should close or not.
1642 @param request: The request most recently received over this channel
1643 against which checks will be made to determine if this connection
1644 can remain open after a matching response is returned.
1646 @type version: C{str}
1647 @param version: The version of the request.
1650 @return: A flag which, if C{True}, indicates that this connection may
1651 remain open to receive another request; if C{False}, the connection
1652 must be closed in order to indicate the completion of the response
1655 connection = request.requestHeaders.getRawHeaders('connection')
1657 tokens = map(str.lower, connection[0].split(' '))
1661 # HTTP 1.0 persistent connection support is currently disabled,
1662 # since we need a way to disable pipelining. HTTP 1.0 can't do
1663 # pipelining since we can't know in advance if we'll have a
1664 # content-length header, if we don't have the header we need to close the
1665 # connection. In HTTP 1.1 this is not an issue since we use chunked
1666 # encoding if content-length is not available.
1668 #if version == "HTTP/1.0":
1669 # if 'keep-alive' in tokens:
1670 # request.setHeader('connection', 'Keep-Alive')
1674 if version == "HTTP/1.1":
1675 if 'close' in tokens:
1676 request.responseHeaders.setRawHeaders('connection', ['close'])
1684 def requestDone(self, request):
1686 Called by first request in queue when it is done.
1688 if request != self.requests[0]: raise TypeError
1689 del self.requests[0]
1692 # notify next request it can start writing
1694 self.requests[0].noLongerQueued()
1696 if self._savedTimeOut:
1697 self.setTimeout(self._savedTimeOut)
1699 self.transport.loseConnection()
1701 def timeoutConnection(self):
1702 log.msg("Timing out client: %s" % str(self.transport.getPeer()))
1703 policies.TimeoutMixin.timeoutConnection(self)
1705 def connectionLost(self, reason):
1706 self.setTimeout(None)
1707 for request in self.requests:
1708 request.connectionLost(reason)
1711 class HTTPFactory(protocol.ServerFactory):
1713 Factory for HTTP server.
1715 @ivar _logDateTime: A cached datetime string for log messages, updated by
1716 C{_logDateTimeCall}.
1717 @type _logDateTime: L{str}
1719 @ivar _logDateTimeCall: A delayed call for the next update to the cached log
1721 @type _logDateTimeCall: L{IDelayedCall} provided
1724 protocol = HTTPChannel
1728 timeOut = 60 * 60 * 12
1730 def __init__(self, logPath=None, timeout=60*60*12):
1731 if logPath is not None:
1732 logPath = os.path.abspath(logPath)
1733 self.logPath = logPath
1734 self.timeOut = timeout
1736 # For storing the cached log datetime and the callback to update it
1737 self._logDateTime = None
1738 self._logDateTimeCall = None
1741 def _updateLogDateTime(self):
1743 Update log datetime periodically, so we aren't always recalculating it.
1745 self._logDateTime = datetimeToLogString()
1746 self._logDateTimeCall = reactor.callLater(1, self._updateLogDateTime)
1749 def buildProtocol(self, addr):
1750 p = protocol.ServerFactory.buildProtocol(self, addr)
1751 # timeOut needs to be on the Protocol instance cause
1752 # TimeoutMixin expects it there
1753 p.timeOut = self.timeOut
1757 def startFactory(self):
1759 Set up request logging if necessary.
1761 if self._logDateTimeCall is None:
1762 self._updateLogDateTime()
1765 self.logFile = self._openLogFile(self.logPath)
1767 self.logFile = log.logfile
1770 def stopFactory(self):
1771 if hasattr(self, "logFile"):
1772 if self.logFile != log.logfile:
1773 self.logFile.close()
1776 if self._logDateTimeCall is not None and self._logDateTimeCall.active():
1777 self._logDateTimeCall.cancel()
1778 self._logDateTimeCall = None
1781 def _openLogFile(self, path):
1783 Override in subclasses, e.g. to use twisted.python.logfile.
1785 f = open(path, "a", 1)
1788 def _escape(self, s):
1789 # pain in the ass. Return a string like python repr, but always
1790 # escaped as if surrounding quotes were "".
1793 return r[1:-1].replace('"', '\\"').replace("\\'", "'")
1796 def log(self, request):
1798 Log a request's result to the logfile, by default in combined log format.
1800 if hasattr(self, "logFile"):
1801 line = '%s - - %s "%s" %d %s "%s" "%s"\n' % (
1802 request.getClientIP(),
1803 # request.getUser() or "-", # the remote user is almost never important
1805 '%s %s %s' % (self._escape(request.method),
1806 self._escape(request.uri),
1807 self._escape(request.clientproto)),
1809 request.sentLength or "-",
1810 self._escape(request.getHeader("referer") or "-"),
1811 self._escape(request.getHeader("user-agent") or "-"))
1812 self.logFile.write(line)