twisted/web/test/test_proxy.py

   1 # Copyright (c) Twisted Matrix Laboratories.
   2 # See LICENSE for details.
   3
   4 """
   5 Test for L{twisted.web.proxy}.
   6 """
   7
   8 from twisted.trial.unittest import TestCase
   9 from twisted.test.proto_helpers import StringTransportWithDisconnection
  10 from twisted.test.proto_helpers import MemoryReactor
  11
  12 from twisted.web.resource import Resource
  13 from twisted.web.server import Site
  14 from twisted.web.proxy import ReverseProxyResource, ProxyClientFactory
  15 from twisted.web.proxy import ProxyClient, ProxyRequest, ReverseProxyRequest
  16 from twisted.web.test.test_web import DummyRequest
  17
  18
  19 class ReverseProxyResourceTestCase(TestCase):
  20     """
  21     Tests for L{ReverseProxyResource}.
  22     """
  23
  24     def _testRender(self, uri, expectedURI):
  25         """
  26         Check that a request pointing at C{uri} produce a new proxy connection,
  27         with the path of this request pointing at C{expectedURI}.
  28         """
  29         root = Resource()
  30         reactor = MemoryReactor()
  31         resource = ReverseProxyResource("127.0.0.1", 1234, "/path", reactor)
  32         root.putChild('index', resource)
  33         site = Site(root)
  34
  35         transport = StringTransportWithDisconnection()
  36         channel = site.buildProtocol(None)
  37         channel.makeConnection(transport)
  38         # Clear the timeout if the tests failed
  39         self.addCleanup(channel.connectionLost, None)
  40
  41         channel.dataReceived("GET %s HTTP/1.1\r\nAccept: text/html\r\n\r\n" %
  42                              (uri,))
  43
  44         # Check that one connection has been created, to the good host/port
  45         self.assertEqual(len(reactor.tcpClients), 1)
  46         self.assertEqual(reactor.tcpClients[0][0], "127.0.0.1")
  47         self.assertEqual(reactor.tcpClients[0][1], 1234)
  48
  49         # Check the factory passed to the connect, and its given path
  50         factory = reactor.tcpClients[0][2]
  51         self.assertIsInstance(factory, ProxyClientFactory)
  52         self.assertEqual(factory.rest, expectedURI)
  53         self.assertEqual(factory.headers["host"], "127.0.0.1:1234")
  54
  55
  56     def test_render(self):
  57         """
  58         Test that L{ReverseProxyResource.render} initiates a connection to the
  59         given server with a L{ProxyClientFactory} as parameter.
  60         """
  61         return self._testRender("/index", "/path")
  62
  63
  64     def test_renderWithQuery(self):
  65         """
  66         Test that L{ReverseProxyResource.render} passes query parameters to the
  67         created factory.
  68         """
  69         return self._testRender("/index?foo=bar", "/path?foo=bar")
  70
  71
  72     def test_getChild(self):
  73         """
  74         The L{ReverseProxyResource.getChild} method should return a resource
  75         instance with the same class as the originating resource, forward
  76         port, host, and reactor values, and update the path value with the
  77         value passed.
  78         """
  79         reactor = MemoryReactor()
  80         resource = ReverseProxyResource("127.0.0.1", 1234, "/path", reactor)
  81         child = resource.getChild('foo', None)
  82         # The child should keep the same class
  83         self.assertIsInstance(child, ReverseProxyResource)
  84         self.assertEqual(child.path, "/path/foo")
  85         self.assertEqual(child.port, 1234)
  86         self.assertEqual(child.host, "127.0.0.1")
  87         self.assertIdentical(child.reactor, resource.reactor)
  88
  89
  90     def test_getChildWithSpecial(self):
  91         """
  92         The L{ReverseProxyResource} return by C{getChild} has a path which has
  93         already been quoted.
  94         """
  95         resource = ReverseProxyResource("127.0.0.1", 1234, "/path")
  96         child = resource.getChild(' /%', None)
  97         self.assertEqual(child.path, "/path/%20%2F%25")
  98
  99
 100
 101 class DummyChannel(object):
 102     """
 103     A dummy HTTP channel, that does nothing but holds a transport and saves
 104     connection lost.
 105
 106     @ivar transport: the transport used by the client.
 107     @ivar lostReason: the reason saved at connection lost.
 108     """
 109
 110     def __init__(self, transport):
 111         """
 112         Hold a reference to the transport.
 113         """
 114         self.transport = transport
 115         self.lostReason = None
 116
 117
 118     def connectionLost(self, reason):
 119         """
 120         Keep track of the connection lost reason.
 121         """
 122         self.lostReason = reason
 123
 124
 125
 126 class ProxyClientTestCase(TestCase):
 127     """
 128     Tests for L{ProxyClient}.
 129     """
 130
 131     def _parseOutHeaders(self, content):
 132         """
 133         Parse the headers out of some web content.
 134
 135         @param content: Bytes received from a web server.
 136         @return: A tuple of (requestLine, headers, body). C{headers} is a dict
 137             of headers, C{requestLine} is the first line (e.g. "POST /foo ...")
 138             and C{body} is whatever is left.
 139         """
 140         headers, body = content.split('\r\n\r\n')
 141         headers = headers.split('\r\n')
 142         requestLine = headers.pop(0)
 143         return (
 144             requestLine, dict(header.split(': ') for header in headers), body)
 145
 146
 147     def makeRequest(self, path):
 148         """
 149         Make a dummy request object for the URL path.
 150
 151         @param path: A URL path, beginning with a slash.
 152         @return: A L{DummyRequest}.
 153         """
 154         return DummyRequest(path)
 155
 156
 157     def makeProxyClient(self, request, method="GET", headers=None,
 158                         requestBody=""):
 159         """
 160         Make a L{ProxyClient} object used for testing.
 161
 162         @param request: The request to use.
 163         @param method: The HTTP method to use, GET by default.
 164         @param headers: The HTTP headers to use expressed as a dict. If not
 165             provided, defaults to {'accept': 'text/html'}.
 166         @param requestBody: The body of the request. Defaults to the empty
 167             string.
 168         @return: A L{ProxyClient}
 169         """
 170         if headers is None:
 171             headers = {"accept": "text/html"}
 172         path = '/' + request.postpath
 173         return ProxyClient(
 174             method, path, 'HTTP/1.0', headers, requestBody, request)
 175
 176
 177     def connectProxy(self, proxyClient):
 178         """
 179         Connect a proxy client to a L{StringTransportWithDisconnection}.
 180
 181         @param proxyClient: A L{ProxyClient}.
 182         @return: The L{StringTransportWithDisconnection}.
 183         """
 184         clientTransport = StringTransportWithDisconnection()
 185         clientTransport.protocol = proxyClient
 186         proxyClient.makeConnection(clientTransport)
 187         return clientTransport
 188
 189
 190     def assertForwardsHeaders(self, proxyClient, requestLine, headers):
 191         """
 192         Assert that C{proxyClient} sends C{headers} when it connects.
 193
 194         @param proxyClient: A L{ProxyClient}.
 195         @param requestLine: The request line we expect to be sent.
 196         @param headers: A dict of headers we expect to be sent.
 197         @return: If the assertion is successful, return the request body as
 198             bytes.
 199         """
 200         self.connectProxy(proxyClient)
 201         requestContent = proxyClient.transport.value()
 202         receivedLine, receivedHeaders, body = self._parseOutHeaders(
 203             requestContent)
 204         self.assertEqual(receivedLine, requestLine)
 205         self.assertEqual(receivedHeaders, headers)
 206         return body
 207
 208
 209     def makeResponseBytes(self, code, message, headers, body):
 210         lines = ["HTTP/1.0 %d %s" % (code, message)]
 211         for header, values in headers:
 212             for value in values:
 213                 lines.append("%s: %s" % (header, value))
 214         lines.extend(['', body])
 215         return '\r\n'.join(lines)
 216
 217
 218     def assertForwardsResponse(self, request, code, message, headers, body):
 219         """
 220         Assert that C{request} has forwarded a response from the server.
 221
 222         @param request: A L{DummyRequest}.
 223         @param code: The expected HTTP response code.
 224         @param message: The expected HTTP message.
 225         @param headers: The expected HTTP headers.
 226         @param body: The expected response body.
 227         """
 228         self.assertEqual(request.responseCode, code)
 229         self.assertEqual(request.responseMessage, message)
 230         receivedHeaders = list(request.responseHeaders.getAllRawHeaders())
 231         receivedHeaders.sort()
 232         expectedHeaders = headers[:]
 233         expectedHeaders.sort()
 234         self.assertEqual(receivedHeaders, expectedHeaders)
 235         self.assertEqual(''.join(request.written), body)
 236
 237
 238     def _testDataForward(self, code, message, headers, body, method="GET",
 239                          requestBody="", loseConnection=True):
 240         """
 241         Build a fake proxy connection, and send C{data} over it, checking that
 242         it's forwarded to the originating request.
 243         """
 244         request = self.makeRequest('foo')
 245         client = self.makeProxyClient(
 246             request, method, {'accept': 'text/html'}, requestBody)
 247
 248         receivedBody = self.assertForwardsHeaders(
 249             client, '%s /foo HTTP/1.0' % (method,),
 250             {'connection': 'close', 'accept': 'text/html'})
 251
 252         self.assertEqual(receivedBody, requestBody)
 253
 254         # Fake an answer
 255         client.dataReceived(
 256             self.makeResponseBytes(code, message, headers, body))
 257
 258         # Check that the response data has been forwarded back to the original
 259         # requester.
 260         self.assertForwardsResponse(request, code, message, headers, body)
 261
 262         # Check that when the response is done, the request is finished.
 263         if loseConnection:
 264             client.transport.loseConnection()
 265
 266         # Even if we didn't call loseConnection, the transport should be
 267         # disconnected.  This lets us not rely on the server to close our
 268         # sockets for us.
 269         self.assertFalse(client.transport.connected)
 270         self.assertEqual(request.finished, 1)
 271
 272
 273     def test_forward(self):
 274         """
 275         When connected to the server, L{ProxyClient} should send the saved
 276         request, with modifications of the headers, and then forward the result
 277         to the parent request.
 278         """
 279         return self._testDataForward(
 280             200, "OK", [("Foo", ["bar", "baz"])], "Some data\r\n")
 281
 282
 283     def test_postData(self):
 284         """
 285         Try to post content in the request, and check that the proxy client
 286         forward the body of the request.
 287         """
 288         return self._testDataForward(
 289             200, "OK", [("Foo", ["bar"])], "Some data\r\n", "POST", "Some content")
 290
 291
 292     def test_statusWithMessage(self):
 293         """
 294         If the response contains a status with a message, it should be
 295         forwarded to the parent request with all the information.
 296         """
 297         return self._testDataForward(
 298             404, "Not Found", [], "")
 299
 300
 301     def test_contentLength(self):
 302         """
 303         If the response contains a I{Content-Length} header, the inbound
 304         request object should still only have C{finish} called on it once.
 305         """
 306         data = "foo bar baz"
 307         return self._testDataForward(
 308             200, "OK", [("Content-Length", [str(len(data))])], data)
 309
 310
 311     def test_losesConnection(self):
 312         """
 313         If the response contains a I{Content-Length} header, the outgoing
 314         connection is closed when all response body data has been received.
 315         """
 316         data = "foo bar baz"
 317         return self._testDataForward(
 318             200, "OK", [("Content-Length", [str(len(data))])], data,
 319             loseConnection=False)
 320
 321
 322     def test_headersCleanups(self):
 323         """
 324         The headers given at initialization should be modified:
 325         B{proxy-connection} should be removed if present, and B{connection}
 326         should be added.
 327         """
 328         client = ProxyClient('GET', '/foo', 'HTTP/1.0',
 329                 {"accept": "text/html", "proxy-connection": "foo"}, '', None)
 330         self.assertEqual(client.headers,
 331                 {"accept": "text/html", "connection": "close"})
 332
 333
 334     def test_keepaliveNotForwarded(self):
 335         """
 336         The proxy doesn't really know what to do with keepalive things from
 337         the remote server, so we stomp over any keepalive header we get from
 338         the client.
 339         """
 340         headers = {
 341             "accept": "text/html",
 342             'keep-alive': '300',
 343             'connection': 'keep-alive',
 344             }
 345         expectedHeaders = headers.copy()
 346         expectedHeaders['connection'] = 'close'
 347         del expectedHeaders['keep-alive']
 348         client = ProxyClient('GET', '/foo', 'HTTP/1.0', headers, '', None)
 349         self.assertForwardsHeaders(
 350             client, 'GET /foo HTTP/1.0', expectedHeaders)
 351
 352
 353     def test_defaultHeadersOverridden(self):
 354         """
 355         L{server.Request} within the proxy sets certain response headers by
 356         default. When we get these headers back from the remote server, the
 357         defaults are overridden rather than simply appended.
 358         """
 359         request = self.makeRequest('foo')
 360         request.responseHeaders.setRawHeaders('server', ['old-bar'])
 361         request.responseHeaders.setRawHeaders('date', ['old-baz'])
 362         request.responseHeaders.setRawHeaders('content-type', ["old/qux"])
 363         client = self.makeProxyClient(request, headers={'accept': 'text/html'})
 364         self.connectProxy(client)
 365         headers = {
 366             'Server': ['bar'],
 367             'Date': ['2010-01-01'],
 368             'Content-Type': ['application/x-baz'],
 369             }
 370         client.dataReceived(
 371             self.makeResponseBytes(200, "OK", headers.items(), ''))
 372         self.assertForwardsResponse(
 373             request, 200, 'OK', headers.items(), '')
 374
 375
 376
 377 class ProxyClientFactoryTestCase(TestCase):
 378     """
 379     Tests for L{ProxyClientFactory}.
 380     """
 381
 382     def test_connectionFailed(self):
 383         """
 384         Check that L{ProxyClientFactory.clientConnectionFailed} produces
 385         a B{501} response to the parent request.
 386         """
 387         request = DummyRequest(['foo'])
 388         factory = ProxyClientFactory('GET', '/foo', 'HTTP/1.0',
 389                                      {"accept": "text/html"}, '', request)
 390
 391         factory.clientConnectionFailed(None, None)
 392         self.assertEqual(request.responseCode, 501)
 393         self.assertEqual(request.responseMessage, "Gateway error")
 394         self.assertEqual(
 395             list(request.responseHeaders.getAllRawHeaders()),
 396             [("Content-Type", ["text/html"])])
 397         self.assertEqual(
 398             ''.join(request.written),
 399             "<H1>Could not connect</H1>")
 400         self.assertEqual(request.finished, 1)
 401
 402
 403     def test_buildProtocol(self):
 404         """
 405         L{ProxyClientFactory.buildProtocol} should produce a L{ProxyClient}
 406         with the same values of attributes (with updates on the headers).
 407         """
 408         factory = ProxyClientFactory('GET', '/foo', 'HTTP/1.0',
 409                                      {"accept": "text/html"}, 'Some data',
 410                                      None)
 411         proto = factory.buildProtocol(None)
 412         self.assertIsInstance(proto, ProxyClient)
 413         self.assertEqual(proto.command, 'GET')
 414         self.assertEqual(proto.rest, '/foo')
 415         self.assertEqual(proto.data, 'Some data')
 416         self.assertEqual(proto.headers,
 417                           {"accept": "text/html", "connection": "close"})
 418
 419
 420
 421 class ProxyRequestTestCase(TestCase):
 422     """
 423     Tests for L{ProxyRequest}.
 424     """
 425
 426     def _testProcess(self, uri, expectedURI, method="GET", data=""):
 427         """
 428         Build a request pointing at C{uri}, and check that a proxied request
 429         is created, pointing a C{expectedURI}.
 430         """
 431         transport = StringTransportWithDisconnection()
 432         channel = DummyChannel(transport)
 433         reactor = MemoryReactor()
 434         request = ProxyRequest(channel, False, reactor)
 435         request.gotLength(len(data))
 436         request.handleContentChunk(data)
 437         request.requestReceived(method, 'http://example.com%s' % (uri,),
 438                                 'HTTP/1.0')
 439
 440         self.assertEqual(len(reactor.tcpClients), 1)
 441         self.assertEqual(reactor.tcpClients[0][0], "example.com")
 442         self.assertEqual(reactor.tcpClients[0][1], 80)
 443
 444         factory = reactor.tcpClients[0][2]
 445         self.assertIsInstance(factory, ProxyClientFactory)
 446         self.assertEqual(factory.command, method)
 447         self.assertEqual(factory.version, 'HTTP/1.0')
 448         self.assertEqual(factory.headers, {'host': 'example.com'})
 449         self.assertEqual(factory.data, data)
 450         self.assertEqual(factory.rest, expectedURI)
 451         self.assertEqual(factory.father, request)
 452
 453
 454     def test_process(self):
 455         """
 456         L{ProxyRequest.process} should create a connection to the given server,
 457         with a L{ProxyClientFactory} as connection factory, with the correct
 458         parameters:
 459             - forward comment, version and data values
 460             - update headers with the B{host} value
 461             - remove the host from the URL
 462             - pass the request as parent request
 463         """
 464         return self._testProcess("/foo/bar", "/foo/bar")
 465
 466
 467     def test_processWithoutTrailingSlash(self):
 468         """
 469         If the incoming request doesn't contain a slash,
 470         L{ProxyRequest.process} should add one when instantiating
 471         L{ProxyClientFactory}.
 472         """
 473         return self._testProcess("", "/")
 474
 475
 476     def test_processWithData(self):
 477         """
 478         L{ProxyRequest.process} should be able to retrieve request body and
 479         to forward it.
 480         """
 481         return self._testProcess(
 482             "/foo/bar", "/foo/bar", "POST", "Some content")
 483
 484
 485     def test_processWithPort(self):
 486         """
 487         Check that L{ProxyRequest.process} correctly parse port in the incoming
 488         URL, and create a outgoing connection with this port.
 489         """
 490         transport = StringTransportWithDisconnection()
 491         channel = DummyChannel(transport)
 492         reactor = MemoryReactor()
 493         request = ProxyRequest(channel, False, reactor)
 494         request.gotLength(0)
 495         request.requestReceived('GET', 'http://example.com:1234/foo/bar',
 496                                 'HTTP/1.0')
 497
 498         # That should create one connection, with the port parsed from the URL
 499         self.assertEqual(len(reactor.tcpClients), 1)
 500         self.assertEqual(reactor.tcpClients[0][0], "example.com")
 501         self.assertEqual(reactor.tcpClients[0][1], 1234)
 502
 503
 504
 505 class DummyFactory(object):
 506     """
 507     A simple holder for C{host} and C{port} information.
 508     """
 509
 510     def __init__(self, host, port):
 511         self.host = host
 512         self.port = port
 513
 514
 515
 516 class ReverseProxyRequestTestCase(TestCase):
 517     """
 518     Tests for L{ReverseProxyRequest}.
 519     """
 520
 521     def test_process(self):
 522         """
 523         L{ReverseProxyRequest.process} should create a connection to its
 524         factory host/port, using a L{ProxyClientFactory} instantiated with the
 525         correct parameters, and particulary set the B{host} header to the
 526         factory host.
 527         """
 528         transport = StringTransportWithDisconnection()
 529         channel = DummyChannel(transport)
 530         reactor = MemoryReactor()
 531         request = ReverseProxyRequest(channel, False, reactor)
 532         request.factory = DummyFactory("example.com", 1234)
 533         request.gotLength(0)
 534         request.requestReceived('GET', '/foo/bar', 'HTTP/1.0')
 535
 536         # Check that one connection has been created, to the good host/port
 537         self.assertEqual(len(reactor.tcpClients), 1)
 538         self.assertEqual(reactor.tcpClients[0][0], "example.com")
 539         self.assertEqual(reactor.tcpClients[0][1], 1234)
 540
 541         # Check the factory passed to the connect, and its headers
 542         factory = reactor.tcpClients[0][2]
 543         self.assertIsInstance(factory, ProxyClientFactory)
 544         self.assertEqual(factory.headers, {'host': 'example.com'})