1 // Copyright 2010-2012 Mikeal Rogers
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 var http = require('http')
18 , url = require('url')
19 , util = require('util')
20 , stream = require('stream')
21 , qs = require('querystring')
22 , mimetypes = require('./mimetypes')
23 , oauth = require('./oauth')
24 , uuid = require('./uuid')
25 , ForeverAgent = require('./forever')
26 , Cookie = require('./vendor/cookie')
27 , CookieJar = require('./vendor/cookie/jar')
28 , cookieJar = new CookieJar
29 , tunnel = require('./tunnel')
30 , aws = require('./aws')
33 if (process.logging) {
34 var log = process.logging('request')
38 https = require('https')
45 function toBase64 (str) {
46 return (new Buffer(str || "", "ascii")).toString("base64")
49 // Hacky fix for pre-0.4.4 https
50 if (https && !https.Agent) {
51 https.Agent = function (options) {
52 http.Agent.call(this, options)
54 util.inherits(https.Agent, http.Agent)
55 https.Agent.prototype._getConnection = function (host, port, cb) {
56 var s = tls.connect(port, host, this.options, function () {
57 // do other checks here?
64 function isReadStream (rs) {
65 if (rs.readable && rs.path && rs.mode) {
72 Object.keys(obj).forEach(function (i) {
78 var isUrl = /^https?:/
82 function Request (options) {
83 stream.Stream.call(this)
87 if (typeof options === 'string') {
88 options = {uri:options}
91 var reserved = Object.keys(Request.prototype)
92 for (var i in options) {
93 if (reserved.indexOf(i) === -1) {
96 if (typeof options[i] === 'function') {
101 options = copy(options)
105 util.inherits(Request, stream.Stream)
106 Request.prototype.init = function (options) {
109 if (!options) options = {}
111 if (!self.pool && self.pool !== false) self.pool = globalPool
113 self.__isRequestRequest = true
115 // Protect against double callback
116 if (!self._callback && self.callback) {
117 self._callback = self.callback
118 self.callback = function () {
119 if (self._callbackCalled) return // Print a warning maybe?
120 self._callback.apply(self, arguments)
121 self._callbackCalled = true
123 self.on('error', self.callback.bind())
124 self.on('complete', self.callback.bind(self, null))
128 // People use this property instead all the time so why not just support it.
134 throw new Error("options.uri is a required argument")
136 if (typeof self.uri == "string") self.uri = url.parse(self.uri)
139 if (typeof self.proxy == 'string') self.proxy = url.parse(self.proxy)
141 // do the HTTP CONNECT dance using koichik/node-tunnel
142 if (http.globalAgent && self.uri.protocol === "https:") {
144 var tunnelFn = self.proxy.protocol === "http:"
145 ? tunnel.httpsOverHttp : tunnel.httpsOverHttps
147 var tunnelOptions = { proxy: { host: self.proxy.hostname
148 , port: +self.proxy.port
149 , proxyAuth: self.proxy.auth }
152 self.agent = tunnelFn(tunnelOptions)
157 if (!self.uri.host || !self.uri.pathname) {
158 // Invalid URI: it may generate lot of bad errors, like "TypeError: Cannot call method 'indexOf' of undefined" in CookieJar
159 // Detect and reject it as soon as possible
160 var faultyUri = url.format(self.uri)
161 var message = 'Invalid URI "' + faultyUri + '"'
162 if (Object.keys(options).length === 0) {
163 // No option ? This can be the sign of a redirect
164 // As this is a case where the user cannot do anything (he didn't call request directly with this URL)
165 // he should be warned that it can be caused by a redirection (can save some hair)
166 message += '. This can be caused by a crappy redirection.'
168 self.emit('error', new Error(message))
169 return // This error was fatal
172 self._redirectsFollowed = self._redirectsFollowed || 0
173 self.maxRedirects = (self.maxRedirects !== undefined) ? self.maxRedirects : 10
174 self.followRedirect = (self.followRedirect !== undefined) ? self.followRedirect : true
175 self.followAllRedirects = (self.followAllRedirects !== undefined) ? self.followAllRedirects : false;
176 if (self.followRedirect || self.followAllRedirects)
177 self.redirects = self.redirects || []
179 self.headers = self.headers ? copy(self.headers) : {}
182 if (!self.headers.host) {
183 self.headers.host = self.uri.hostname
185 if ( !(self.uri.port === 80 && self.uri.protocol === 'http:') &&
186 !(self.uri.port === 443 && self.uri.protocol === 'https:') )
187 self.headers.host += (':'+self.uri.port)
192 self.jar(self._jar || options.jar)
194 if (!self.uri.pathname) {self.uri.pathname = '/'}
195 if (!self.uri.port) {
196 if (self.uri.protocol == 'http:') {self.uri.port = 80}
197 else if (self.uri.protocol == 'https:') {self.uri.port = 443}
200 if (self.proxy && !self.tunnel) {
201 self.port = self.proxy.port
202 self.host = self.proxy.hostname
204 self.port = self.uri.port
205 self.host = self.uri.hostname
208 self.clientErrorHandler = function (error) {
209 if (self._aborted) return
211 if (self.setHost) delete self.headers.host
212 if (self.req._reusedSocket && error.code === 'ECONNRESET'
213 && self.agent.addRequestNoreuse) {
214 self.agent = { addRequest: self.agent.addRequestNoreuse.bind(self.agent) }
219 if (self.timeout && self.timeoutTimer) {
220 clearTimeout(self.timeoutTimer)
221 self.timeoutTimer = null
223 self.emit('error', error)
227 self.form(options.form)
231 self.oauth(options.oauth)
235 self.aws(options.aws)
238 if (self.uri.auth && !self.headers.authorization) {
239 self.headers.authorization = "Basic " + toBase64(self.uri.auth.split(':').map(function(item){ return qs.unescape(item)}).join(':'))
241 if (self.proxy && self.proxy.auth && !self.headers['proxy-authorization'] && !self.tunnel) {
242 self.headers['proxy-authorization'] = "Basic " + toBase64(self.proxy.auth.split(':').map(function(item){ return qs.unescape(item)}).join(':'))
245 if (options.qs) self.qs(options.qs)
248 self.path = self.uri.path
250 self.path = self.uri.pathname + (self.uri.search || "")
253 if (self.path.length === 0) self.path = '/'
255 if (self.proxy && !self.tunnel) self.path = (self.uri.protocol + '//' + self.uri.host + self.path)
258 self.json(options.json)
259 } else if (options.multipart) {
260 self.boundary = uuid()
261 self.multipart(options.multipart)
266 if (!Buffer.isBuffer(self.body)) {
267 if (Array.isArray(self.body)) {
268 for (var i = 0; i < self.body.length; i++) {
269 length += self.body[i].length
272 self.body = new Buffer(self.body)
273 length = self.body.length
276 length = self.body.length
279 self.headers['content-length'] = length
281 throw new Error('Argument error, options.body.')
285 var protocol = self.proxy && !self.tunnel ? self.proxy.protocol : self.uri.protocol
286 , defaultModules = {'http:':http, 'https:':https}
287 , httpModules = self.httpModules || {}
289 self.httpModule = httpModules[protocol] || defaultModules[protocol]
291 if (!self.httpModule) throw new Error("Invalid protocol")
293 if (options.ca) self.ca = options.ca
296 if (options.agentOptions) self.agentOptions = options.agentOptions
298 if (options.agentClass) {
299 self.agentClass = options.agentClass
300 } else if (options.forever) {
301 self.agentClass = protocol === 'http:' ? ForeverAgent : ForeverAgent.SSL
303 self.agentClass = self.httpModule.Agent
307 if (self.pool === false) {
310 self.agent = self.agent || self.getAgent()
311 if (self.maxSockets) {
312 // Don't use our pooling if node has the refactored client
313 self.agent.maxSockets = self.maxSockets
315 if (self.pool.maxSockets) {
316 // Don't use our pooling if node has the refactored client
317 self.agent.maxSockets = self.pool.maxSockets
321 self.once('pipe', function (src) {
322 if (self.ntick) throw new Error("You cannot pipe to this stream after the first nextTick() after creation of the request stream.")
324 if (isReadStream(src)) {
325 if (!self.headers['content-type'] && !self.headers['Content-Type'])
326 self.headers['content-type'] = mimetypes.lookup(src.path.slice(src.path.lastIndexOf('.')+1))
329 for (var i in src.headers) {
330 if (!self.headers[i]) {
331 self.headers[i] = src.headers[i]
335 if (src.method && !self.method) {
336 self.method = src.method
340 self.on('pipe', function () {
341 console.error("You have already piped to this stream. Pipeing twice is likely to break the request.")
345 process.nextTick(function () {
346 if (self._aborted) return
349 if (Array.isArray(self.body)) {
350 self.body.forEach(function (part) {
354 self.write(self.body)
357 } else if (self.requestBodyStream) {
358 console.warn("options.requestBodyStream is deprecated, please pass the request object to stream.pipe.")
359 self.requestBodyStream.pipe(self)
360 } else if (!self.src) {
361 if (self.method !== 'GET' && typeof self.method !== 'undefined') {
362 self.headers['content-length'] = 0;
370 Request.prototype.getAgent = function () {
371 var Agent = this.agentClass
373 if (this.agentOptions) {
374 for (var i in this.agentOptions) {
375 options[i] = this.agentOptions[i]
378 if (this.ca) options.ca = this.ca
382 // different types of agents are in different pools
383 if (Agent !== this.httpModule.Agent) {
384 poolKey += Agent.name
387 if (!this.httpModule.globalAgent) {
389 options.host = this.host
390 options.port = this.port
391 if (poolKey) poolKey += ':'
392 poolKey += this.host + ':' + this.port
396 if (poolKey) poolKey += ':'
397 poolKey += options.ca
400 if (!poolKey && Agent === this.httpModule.Agent && this.httpModule.globalAgent) {
401 // not doing anything special. Use the globalAgent
402 return this.httpModule.globalAgent
405 // already generated an agent for this setting
406 if (this.pool[poolKey]) return this.pool[poolKey]
408 return this.pool[poolKey] = new Agent(options)
411 Request.prototype.start = function () {
414 if (self._aborted) return
417 self.method = self.method || 'GET'
418 self.href = self.uri.href
419 if (log) log('%method %href', self)
421 if (self.src && self.src.stat && self.src.stat.size) {
422 self.headers['content-length'] = self.src.stat.size
425 self.aws(self._aws, true)
428 self.req = self.httpModule.request(self, function (response) {
429 if (self._aborted) return
430 if (self._paused) response.pause()
432 self.response = response
433 response.request = self
434 response.toJSON = toJSON
436 if (self.httpModule === https &&
438 !response.client.authorized) {
439 var sslErr = response.client.authorizationError
440 self.emit('error', new Error('SSL Error: '+ sslErr))
444 if (self.setHost) delete self.headers.host
445 if (self.timeout && self.timeoutTimer) {
446 clearTimeout(self.timeoutTimer)
447 self.timeoutTimer = null
450 var addCookie = function (cookie) {
451 if (self._jar) self._jar.add(new Cookie(cookie))
452 else cookieJar.add(new Cookie(cookie))
455 if (response.headers['set-cookie'] && (!self._disableCookies)) {
456 if (Array.isArray(response.headers['set-cookie'])) response.headers['set-cookie'].forEach(addCookie)
457 else addCookie(response.headers['set-cookie'])
460 if (response.statusCode >= 300 && response.statusCode < 400 &&
461 (self.followAllRedirects ||
462 (self.followRedirect && (self.method !== 'PUT' && self.method !== 'POST' && self.method !== 'DELETE'))) &&
463 response.headers.location) {
464 if (self._redirectsFollowed >= self.maxRedirects) {
465 self.emit('error', new Error("Exceeded maxRedirects. Probably stuck in a redirect loop."))
468 self._redirectsFollowed += 1
470 if (!isUrl.test(response.headers.location)) {
471 response.headers.location = url.resolve(self.uri.href, response.headers.location)
473 self.uri = response.headers.location
475 { statusCode : response.statusCode
476 , redirectUri: response.headers.location
479 if (self.followAllRedirects) self.method = 'GET'
480 // self.method = 'GET'; // Force all redirects to use GET || commented out fixes #215
486 delete self.headers.host
488 if (log) log('Redirect to %uri', self)
490 return // Ignore the rest of the response
492 self._redirectsFollowed = self._redirectsFollowed || 0
493 // Be a good stream and emit end when the response is finished.
494 // Hack to emit end on close because of a core bug that never fires end
495 response.on('close', function () {
496 if (!self._ended) self.response.emit('end')
500 if (self.dests.length !== 0) {
501 console.error("Ingoring encoding parameter as this stream is being piped to another stream which makes the encoding option invalid.")
503 response.setEncoding(self.encoding)
507 self.dests.forEach(function (dest) {
511 response.on("data", function (chunk) {
512 self._destdata = true
513 self.emit("data", chunk)
515 response.on("end", function (chunk) {
517 self.emit("end", chunk)
519 response.on("close", function () {self.emit("close")})
521 self.emit('response', response)
526 self.on("data", function (chunk) {
528 bodyLen += chunk.length
530 self.on("end", function () {
531 if (self._aborted) return
533 if (buffer.length && Buffer.isBuffer(buffer[0])) {
534 var body = new Buffer(bodyLen)
536 buffer.forEach(function (chunk) {
537 chunk.copy(body, i, 0, chunk.length)
540 if (self.encoding === null) {
543 response.body = body.toString()
545 } else if (buffer.length) {
546 response.body = buffer.join('')
551 response.body = JSON.parse(response.body)
555 self.emit('complete', response, response.body)
561 if (self.timeout && !self.timeoutTimer) {
562 self.timeoutTimer = setTimeout(function () {
564 var e = new Error("ETIMEDOUT")
566 self.emit("error", e)
569 // Set additional timeout on socket - in case if remote
570 // server freeze after sending headers
571 if (self.req.setTimeout) { // only works on node 0.6+
572 self.req.setTimeout(self.timeout, function () {
575 var e = new Error("ESOCKETTIMEDOUT")
576 e.code = "ESOCKETTIMEDOUT"
577 self.emit("error", e)
583 self.req.on('error', self.clientErrorHandler)
584 self.req.on('drain', function() {
588 self.emit('request', self.req)
591 Request.prototype.abort = function () {
592 this._aborted = true;
597 else if (this.response) {
598 this.response.abort()
604 Request.prototype.pipeDest = function (dest) {
605 var response = this.response
606 // Called after the response is received
608 dest.headers['content-type'] = response.headers['content-type']
609 if (response.headers['content-length']) {
610 dest.headers['content-length'] = response.headers['content-length']
613 if (dest.setHeader) {
614 for (var i in response.headers) {
615 dest.setHeader(i, response.headers[i])
617 dest.statusCode = response.statusCode
619 if (this.pipefilter) this.pipefilter(response, dest)
623 Request.prototype.setHeader = function (name, value, clobber) {
624 if (clobber === undefined) clobber = true
625 if (clobber || !this.headers.hasOwnProperty(name)) this.headers[name] = value
626 else this.headers[name] += ',' + value
629 Request.prototype.setHeaders = function (headers) {
630 for (var i in headers) {this.setHeader(i, headers[i])}
633 Request.prototype.qs = function (q, clobber) {
635 if (!clobber && this.uri.query) base = qs.parse(this.uri.query)
642 this.uri = url.parse(this.uri.href.split('?')[0] + '?' + qs.stringify(base))
647 Request.prototype.form = function (form) {
648 this.headers['content-type'] = 'application/x-www-form-urlencoded; charset=utf-8'
649 this.body = qs.stringify(form).toString('utf8')
652 Request.prototype.multipart = function (multipart) {
656 if (!self.headers['content-type']) {
657 self.headers['content-type'] = 'multipart/related; boundary=' + self.boundary;
659 self.headers['content-type'] = self.headers['content-type'].split(';')[0] + '; boundary=' + self.boundary;
662 console.log('boundary >> ' + self.boundary)
664 if (!multipart.forEach) throw new Error('Argument error, options.multipart.')
666 multipart.forEach(function (part) {
668 if(body == null) throw Error('Body attribute missing in multipart.')
670 var preamble = '--' + self.boundary + '\r\n'
671 Object.keys(part).forEach(function (key) {
672 preamble += key + ': ' + part[key] + '\r\n'
675 self.body.push(new Buffer(preamble))
676 self.body.push(new Buffer(body))
677 self.body.push(new Buffer('\r\n'))
679 self.body.push(new Buffer('--' + self.boundary + '--'))
682 Request.prototype.json = function (val) {
683 this.setHeader('content-type', 'application/json')
684 this.setHeader('accept', 'application/json')
686 if (typeof val === 'boolean') {
687 if (typeof this.body === 'object') this.body = JSON.stringify(this.body)
689 this.body = JSON.stringify(val)
693 Request.prototype.aws = function (opts, now) {
698 var date = new Date()
699 this.setHeader('date', date.toUTCString())
700 this.setHeader('authorization', aws.authorization(
702 , secret: opts.secret
705 , resource: aws.canonicalizeResource('/' + opts.bucket + this.path)
706 , contentType: this.headers['content-type'] || ''
707 , md5: this.headers['content-md5'] || ''
708 , amazonHeaders: aws.canonicalizeHeaders(this.headers)
715 Request.prototype.oauth = function (_oauth) {
717 if (this.headers['content-type'] &&
718 this.headers['content-type'].slice(0, 'application/x-www-form-urlencoded'.length) ===
719 'application/x-www-form-urlencoded'
721 form = qs.parse(this.body)
723 if (this.uri.query) {
724 form = qs.parse(this.uri.query)
728 for (var i in form) oa[i] = form[i]
729 for (var i in _oauth) oa['oauth_'+i] = _oauth[i]
730 if (!oa.oauth_version) oa.oauth_version = '1.0'
731 if (!oa.oauth_timestamp) oa.oauth_timestamp = Math.floor( (new Date()).getTime() / 1000 ).toString()
732 if (!oa.oauth_nonce) oa.oauth_nonce = uuid().replace(/-/g, '')
734 oa.oauth_signature_method = 'HMAC-SHA1'
736 var consumer_secret = oa.oauth_consumer_secret
737 delete oa.oauth_consumer_secret
738 var token_secret = oa.oauth_token_secret
739 delete oa.oauth_token_secret
741 var baseurl = this.uri.protocol + '//' + this.uri.host + this.uri.pathname
742 var signature = oauth.hmacsign(this.method, baseurl, oa, consumer_secret, token_secret)
744 // oa.oauth_signature = signature
745 for (var i in form) {
746 if ( i.slice(0, 'oauth_') in _oauth) {
749 delete oa['oauth_'+i]
752 this.headers.Authorization =
753 'OAuth '+Object.keys(oa).sort().map(function (i) {return i+'="'+oauth.rfc3986(oa[i])+'"'}).join(',')
754 this.headers.Authorization += ',oauth_signature="'+oauth.rfc3986(signature)+'"'
757 Request.prototype.jar = function (jar) {
760 if (this._redirectsFollowed === 0) {
761 this.originalCookieHeader = this.headers.cookie
767 this._disableCookies = true;
769 // fetch cookie from the user defined cookie jar
770 cookies = jar.get({ url: this.uri.href })
772 // fetch cookie from the global cookie jar
773 cookies = cookieJar.get({ url: this.uri.href })
776 if (cookies && cookies.length) {
777 var cookieString = cookies.map(function (c) {
778 return c.name + "=" + c.value
781 if (this.originalCookieHeader) {
782 // Don't overwrite existing Cookie header
783 this.headers.cookie = this.originalCookieHeader + '; ' + cookieString
785 this.headers.cookie = cookieString
794 Request.prototype.pipe = function (dest, opts) {
796 if (this._destdata) {
797 throw new Error("You cannot pipe after data has been emitted from the response.")
798 } else if (this._ended) {
799 throw new Error("You cannot pipe after the response has been ended.")
801 stream.Stream.prototype.pipe.call(this, dest, opts)
806 this.dests.push(dest)
807 stream.Stream.prototype.pipe.call(this, dest, opts)
811 Request.prototype.write = function () {
812 if (!this._started) this.start()
813 return this.req.write.apply(this.req, arguments)
815 Request.prototype.end = function (chunk) {
816 if (chunk) this.write(chunk)
817 if (!this._started) this.start()
820 Request.prototype.pause = function () {
821 if (!this.response) this._paused = true
822 else this.response.pause.apply(this.response, arguments)
824 Request.prototype.resume = function () {
825 if (!this.response) this._paused = false
826 else this.response.resume.apply(this.response, arguments)
828 Request.prototype.destroy = function () {
829 if (!this._ended) this.end()
832 // organize params for post, put, head, del
833 function initParams(uri, options, callback) {
834 if ((typeof options === 'function') && !callback) callback = options;
835 if (options && typeof options === 'object') {
837 } else if (typeof uri === 'string') {
843 return { uri: uri, options: options, callback: callback };
846 function request (uri, options, callback) {
847 if (typeof uri === 'undefined') throw new Error('undefined is not a valid uri or options object.')
848 if ((typeof options === 'function') && !callback) callback = options;
849 if (options && typeof options === 'object') {
851 } else if (typeof uri === 'string') {
857 if (callback) options.callback = callback;
858 var r = new Request(options)
862 module.exports = request
864 request.defaults = function (options) {
865 var def = function (method) {
866 var d = function (uri, opts, callback) {
867 var params = initParams(uri, opts, callback);
868 for (var i in options) {
869 if (params.options[i] === undefined) params.options[i] = options[i]
871 return method(params.options, params.callback)
875 var de = def(request)
876 de.get = def(request.get)
877 de.post = def(request.post)
878 de.put = def(request.put)
879 de.head = def(request.head)
880 de.del = def(request.del)
881 de.cookie = def(request.cookie)
882 de.jar = def(request.jar)
886 request.forever = function (agentOptions, optionsArg) {
889 for (option in optionsArg) {
890 options[option] = optionsArg[option]
893 if (agentOptions) options.agentOptions = agentOptions
894 options.forever = true
895 return request.defaults(options)
898 request.get = request
899 request.post = function (uri, options, callback) {
900 var params = initParams(uri, options, callback);
901 params.options.method = 'POST';
902 return request(params.uri || null, params.options, params.callback)
904 request.put = function (uri, options, callback) {
905 var params = initParams(uri, options, callback);
906 params.options.method = 'PUT'
907 return request(params.uri || null, params.options, params.callback)
909 request.head = function (uri, options, callback) {
910 var params = initParams(uri, options, callback);
911 params.options.method = 'HEAD'
912 if (params.options.body ||
913 params.options.requestBodyStream ||
914 (params.options.json && typeof params.options.json !== 'boolean') ||
915 params.options.multipart) {
916 throw new Error("HTTP HEAD requests MUST NOT include a request body.")
918 return request(params.uri || null, params.options, params.callback)
920 request.del = function (uri, options, callback) {
921 var params = initParams(uri, options, callback);
922 params.options.method = 'DELETE'
923 return request(params.uri || null, params.options, params.callback)
925 request.jar = function () {
928 request.cookie = function (str) {
929 if (str && str.uri) str = str.uri
930 if (typeof str !== 'string') throw new Error("The cookie function only accepts STRING as param")
931 return new Cookie(str)
936 function getSafe (self, uuid) {
937 if (typeof self === 'object' || typeof self === 'function') var safe = {}
938 if (Array.isArray(self)) var safe = []
942 Object.defineProperty(self, uuid, {})
944 var attrs = Object.keys(self).filter(function (i) {
945 if (i === uuid) return false
946 if ( (typeof self[i] !== 'object' && typeof self[i] !== 'function') || self[i] === null) return true
947 return !(Object.getOwnPropertyDescriptor(self[i], uuid))
951 for (var i=0;i<attrs.length;i++) {
952 if ( (typeof self[attrs[i]] !== 'object' && typeof self[attrs[i]] !== 'function') ||
953 self[attrs[i]] === null
955 safe[attrs[i]] = self[attrs[i]]
957 recurse.push(attrs[i])
958 Object.defineProperty(self[attrs[i]], uuid, {})
962 for (var i=0;i<recurse.length;i++) {
963 safe[recurse[i]] = getSafe(self[recurse[i]], uuid)
970 return getSafe(this, (((1+Math.random())*0x10000)|0).toString(16))
973 Request.prototype.toJSON = toJSON