1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // HTTP Request reading and parsing.
29 chunkSize = 4 << 10 // 4 KB chunks
30 defaultMaxMemory = 32 << 20 // 32 MB
33 // ErrMissingFile is returned by FormFile when the provided file field name
34 // is either not present in the request or not a file field.
35 var ErrMissingFile = errors.New("http: no such file")
37 // HTTP request parsing errors.
38 type ProtocolError struct {
42 func (err *ProtocolError) Error() string { return err.ErrorString }
45 ErrHeaderTooLong = &ProtocolError{"header too long"}
46 ErrShortBody = &ProtocolError{"entity body too short"}
47 ErrNotSupported = &ProtocolError{"feature not supported"}
48 ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"}
49 ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"}
50 ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"}
51 ErrMissingBoundary = &ProtocolError{"no multipart boundary param Content-Type"}
54 type badStringError struct {
59 func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) }
61 // Headers that Request.Write handles itself and should be skipped.
62 var reqWriteExcludeHeader = map[string]bool{
63 "Host": true, // not in Header map anyway
65 "Content-Length": true,
66 "Transfer-Encoding": true,
70 // A Request represents an HTTP request received by a server
71 // or to be sent by a client.
73 Method string // GET, POST, PUT, etc.
75 // URL is created from the URI supplied on the Request-Line
76 // as stored in RequestURI.
78 // For most requests, fields other than Path and RawQuery
79 // will be empty. (See RFC 2616, Section 5.1.2)
82 // The protocol version for incoming requests.
83 // Outgoing requests always use HTTP/1.1.
84 Proto string // "HTTP/1.0"
88 // A header maps request lines to their values.
91 // accept-encoding: gzip, deflate
92 // Accept-Language: en-us
93 // Connection: keep-alive
97 // Header = map[string][]string{
98 // "Accept-Encoding": {"gzip, deflate"},
99 // "Accept-Language": {"en-us"},
100 // "Connection": {"keep-alive"},
103 // HTTP defines that header names are case-insensitive.
104 // The request parser implements this by canonicalizing the
105 // name, making the first character and any characters
106 // following a hyphen uppercase and the rest lowercase.
112 // ContentLength records the length of the associated content.
113 // The value -1 indicates that the length is unknown.
114 // Values >= 0 indicate that the given number of bytes may
115 // be read from Body.
116 // For outgoing requests, a value of 0 means unknown if Body is not nil.
119 // TransferEncoding lists the transfer encodings from outermost to
120 // innermost. An empty list denotes the "identity" encoding.
121 // TransferEncoding can usually be ignored; chunked encoding is
122 // automatically added and removed as necessary when sending and
123 // receiving requests.
124 TransferEncoding []string
126 // Close indicates whether to close the connection after
127 // replying to this request.
130 // The host on which the URL is sought.
131 // Per RFC 2616, this is either the value of the Host: header
132 // or the host name given in the URL itself.
133 // It may be of the form "host:port".
136 // Form contains the parsed form data, including both the URL
137 // field's query parameters and the POST or PUT form data.
138 // This field is only available after ParseForm is called.
139 // The HTTP client ignores Form and uses Body instead.
142 // PostForm contains the parsed form data from POST or PUT
144 // This field is only available after ParseForm is called.
145 // The HTTP client ignores PostForm and uses Body instead.
148 // MultipartForm is the parsed multipart form, including file uploads.
149 // This field is only available after ParseMultipartForm is called.
150 // The HTTP client ignores MultipartForm and uses Body instead.
151 MultipartForm *multipart.Form
153 // Trailer maps trailer keys to values. Like for Header, if the
154 // response has multiple trailer lines with the same key, they will be
155 // concatenated, delimited by commas.
156 // For server requests, Trailer is only populated after Body has been
157 // closed or fully consumed.
158 // Trailer support is only partially complete.
161 // RemoteAddr allows HTTP servers and other software to record
162 // the network address that sent the request, usually for
163 // logging. This field is not filled in by ReadRequest and
164 // has no defined format. The HTTP server in this package
165 // sets RemoteAddr to an "IP:port" address before invoking a
167 // This field is ignored by the HTTP client.
170 // RequestURI is the unmodified Request-URI of the
171 // Request-Line (RFC 2616, Section 5.1) as sent by the client
172 // to a server. Usually the URL field should be used instead.
173 // It is an error to set this field in an HTTP client request.
176 // TLS allows HTTP servers and other software to record
177 // information about the TLS connection on which the request
178 // was received. This field is not filled in by ReadRequest.
179 // The HTTP server in this package sets the field for
180 // TLS-enabled connections before invoking a handler;
181 // otherwise it leaves the field nil.
182 // This field is ignored by the HTTP client.
183 TLS *tls.ConnectionState
186 // ProtoAtLeast returns whether the HTTP protocol used
187 // in the request is at least major.minor.
188 func (r *Request) ProtoAtLeast(major, minor int) bool {
189 return r.ProtoMajor > major ||
190 r.ProtoMajor == major && r.ProtoMinor >= minor
193 // UserAgent returns the client's User-Agent, if sent in the request.
194 func (r *Request) UserAgent() string {
195 return r.Header.Get("User-Agent")
198 // Cookies parses and returns the HTTP cookies sent with the request.
199 func (r *Request) Cookies() []*Cookie {
200 return readCookies(r.Header, "")
203 var ErrNoCookie = errors.New("http: named cookie not present")
205 // Cookie returns the named cookie provided in the request or
206 // ErrNoCookie if not found.
207 func (r *Request) Cookie(name string) (*Cookie, error) {
208 for _, c := range readCookies(r.Header, name) {
211 return nil, ErrNoCookie
214 // AddCookie adds a cookie to the request. Per RFC 6265 section 5.4,
215 // AddCookie does not attach more than one Cookie header field. That
216 // means all cookies, if any, are written into the same line,
217 // separated by semicolon.
218 func (r *Request) AddCookie(c *Cookie) {
219 s := fmt.Sprintf("%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value))
220 if c := r.Header.Get("Cookie"); c != "" {
221 r.Header.Set("Cookie", c+"; "+s)
223 r.Header.Set("Cookie", s)
227 // Referer returns the referring URL, if sent in the request.
229 // Referer is misspelled as in the request itself, a mistake from the
230 // earliest days of HTTP. This value can also be fetched from the
231 // Header map as Header["Referer"]; the benefit of making it available
232 // as a method is that the compiler can diagnose programs that use the
233 // alternate (correct English) spelling req.Referrer() but cannot
234 // diagnose programs that use Header["Referrer"].
235 func (r *Request) Referer() string {
236 return r.Header.Get("Referer")
239 // multipartByReader is a sentinel value.
240 // Its presence in Request.MultipartForm indicates that parsing of the request
241 // body has been handed off to a MultipartReader instead of ParseMultipartFrom.
242 var multipartByReader = &multipart.Form{
243 Value: make(map[string][]string),
244 File: make(map[string][]*multipart.FileHeader),
247 // MultipartReader returns a MIME multipart reader if this is a
248 // multipart/form-data POST request, else returns nil and an error.
249 // Use this function instead of ParseMultipartForm to
250 // process the request body as a stream.
251 func (r *Request) MultipartReader() (*multipart.Reader, error) {
252 if r.MultipartForm == multipartByReader {
253 return nil, errors.New("http: MultipartReader called twice")
255 if r.MultipartForm != nil {
256 return nil, errors.New("http: multipart handled by ParseMultipartForm")
258 r.MultipartForm = multipartByReader
259 return r.multipartReader()
262 func (r *Request) multipartReader() (*multipart.Reader, error) {
263 v := r.Header.Get("Content-Type")
265 return nil, ErrNotMultipart
267 d, params, err := mime.ParseMediaType(v)
268 if err != nil || d != "multipart/form-data" {
269 return nil, ErrNotMultipart
271 boundary, ok := params["boundary"]
273 return nil, ErrMissingBoundary
275 return multipart.NewReader(r.Body, boundary), nil
278 // Return value if nonempty, def otherwise.
279 func valueOrDefault(value, def string) string {
286 const defaultUserAgent = "Go http package"
288 // Write writes an HTTP/1.1 request -- header and body -- in wire format.
289 // This method consults the following fields of the request:
292 // Method (defaults to "GET")
298 // If Body is present, Content-Length is <= 0 and TransferEncoding
299 // hasn't been set to "identity", Write adds "Transfer-Encoding:
300 // chunked" to the header. Body is closed after it is sent.
301 func (r *Request) Write(w io.Writer) error {
302 return r.write(w, false, nil)
305 // WriteProxy is like Write but writes the request in the form
306 // expected by an HTTP proxy. In particular, WriteProxy writes the
307 // initial Request-URI line of the request with an absolute URI, per
308 // section 5.1.2 of RFC 2616, including the scheme and host.
309 // In either case, WriteProxy also writes a Host header, using
310 // either r.Host or r.URL.Host.
311 func (r *Request) WriteProxy(w io.Writer) error {
312 return r.write(w, true, nil)
315 // extraHeaders may be nil
316 func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header) error {
320 return errors.New("http: Request.Write on Request with no Host or URL set")
325 ruri := req.URL.RequestURI()
326 if usingProxy && req.URL.Scheme != "" && req.URL.Opaque == "" {
327 ruri = req.URL.Scheme + "://" + host + ruri
328 } else if req.Method == "CONNECT" && req.URL.Path == "" {
329 // CONNECT requests normally give just the host and port, not a full URL.
332 // TODO(bradfitz): escape at least newlines in ruri?
334 // Wrap the writer in a bufio Writer if it's not already buffered.
335 // Don't always call NewWriter, as that forces a bytes.Buffer
336 // and other small bufio Writers to have a minimum 4k buffer
339 if _, ok := w.(io.ByteWriter); !ok {
340 bw = bufio.NewWriter(w)
344 fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri)
347 fmt.Fprintf(w, "Host: %s\r\n", host)
349 // Use the defaultUserAgent unless the Header contains one, which
350 // may be blank to not send the header.
351 userAgent := defaultUserAgent
352 if req.Header != nil {
353 if ua := req.Header["User-Agent"]; len(ua) > 0 {
358 fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent)
361 // Process Body,ContentLength,Close,Trailer
362 tw, err := newTransferWriter(req)
366 err = tw.WriteHeader(w)
371 // TODO: split long values? (If so, should share code with Conn.Write)
372 err = req.Header.WriteSubset(w, reqWriteExcludeHeader)
377 if extraHeaders != nil {
378 err = extraHeaders.Write(w)
384 io.WriteString(w, "\r\n")
386 // Write body and trailer
387 err = tw.WriteBody(w)
398 // ParseHTTPVersion parses a HTTP version string.
399 // "HTTP/1.0" returns (1, 0, true).
400 func ParseHTTPVersion(vers string) (major, minor int, ok bool) {
401 const Big = 1000000 // arbitrary upper bound
408 if !strings.HasPrefix(vers, "HTTP/") {
411 dot := strings.Index(vers, ".")
415 major, err := strconv.Atoi(vers[5:dot])
416 if err != nil || major < 0 || major > Big {
419 minor, err = strconv.Atoi(vers[dot+1:])
420 if err != nil || minor < 0 || minor > Big {
423 return major, minor, true
426 // NewRequest returns a new Request given a method, URL, and optional body.
427 func NewRequest(method, urlStr string, body io.Reader) (*Request, error) {
428 u, err := url.Parse(urlStr)
432 rc, ok := body.(io.ReadCloser)
433 if !ok && body != nil {
434 rc = ioutil.NopCloser(body)
442 Header: make(Header),
447 switch v := body.(type) {
449 req.ContentLength = int64(v.Len())
451 req.ContentLength = int64(v.Len())
452 case *strings.Reader:
453 req.ContentLength = int64(v.Len())
460 // SetBasicAuth sets the request's Authorization header to use HTTP
461 // Basic Authentication with the provided username and password.
463 // With HTTP Basic Authentication the provided username and password
464 // are not encrypted.
465 func (r *Request) SetBasicAuth(username, password string) {
466 s := username + ":" + password
467 r.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(s)))
470 // ReadRequest reads and parses a request from b.
471 func ReadRequest(b *bufio.Reader) (req *Request, err error) {
473 tp := textproto.NewReader(b)
476 // First line: GET /index.html HTTP/1.0
478 if s, err = tp.ReadLine(); err != nil {
483 err = io.ErrUnexpectedEOF
488 if f = strings.SplitN(s, " ", 3); len(f) < 3 {
489 return nil, &badStringError{"malformed HTTP request", s}
491 req.Method, req.RequestURI, req.Proto = f[0], f[1], f[2]
492 rawurl := req.RequestURI
494 if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok {
495 return nil, &badStringError{"malformed HTTP version", req.Proto}
498 // CONNECT requests are used two different ways, and neither uses a full URL:
499 // The standard use is to tunnel HTTPS through an HTTP proxy.
500 // It looks like "CONNECT www.google.com:443 HTTP/1.1", and the parameter is
501 // just the authority section of a URL. This information should go in req.URL.Host.
503 // The net/rpc package also uses CONNECT, but there the parameter is a path
504 // that starts with a slash. It can be parsed with the regular URL parser,
505 // and the path will end up in req.URL.Path, where it needs to be in order for
507 justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/")
509 rawurl = "http://" + rawurl
512 if req.URL, err = url.ParseRequestURI(rawurl); err != nil {
517 // Strip the bogus "http://" back off.
521 // Subsequent lines: Key: value.
522 mimeHeader, err := tp.ReadMIMEHeader()
526 req.Header = Header(mimeHeader)
528 // RFC2616: Must treat
529 // GET /index.html HTTP/1.1
530 // Host: www.google.com
532 // GET http://www.google.com/index.html HTTP/1.1
533 // Host: doesntmatter
534 // the same. In the second case, any Host line is ignored.
535 req.Host = req.URL.Host
537 req.Host = req.Header.get("Host")
539 delete(req.Header, "Host")
541 fixPragmaCacheControl(req.Header)
543 // TODO: Parse specific header values:
557 // If-Unmodified-Since
559 // Proxy-Authorization
561 // TE (transfer-codings)
569 err = readTransfer(req, b)
577 // MaxBytesReader is similar to io.LimitReader but is intended for
578 // limiting the size of incoming request bodies. In contrast to
579 // io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a
580 // non-EOF error for a Read beyond the limit, and Closes the
581 // underlying reader when its Close method is called.
583 // MaxBytesReader prevents clients from accidentally or maliciously
584 // sending a large request and wasting server resources.
585 func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser {
586 return &maxBytesReader{w: w, r: r, n: n}
589 type maxBytesReader struct {
591 r io.ReadCloser // underlying reader
592 n int64 // max bytes remaining
596 func (l *maxBytesReader) Read(p []byte) (n int, err error) {
600 if res, ok := l.w.(*response); ok {
601 res.requestTooLarge()
604 return 0, errors.New("http: request body too large")
606 if int64(len(p)) > l.n {
614 func (l *maxBytesReader) Close() error {
618 func copyValues(dst, src url.Values) {
619 for k, vs := range src {
620 for _, value := range vs {
626 func parsePostForm(r *Request) (vs url.Values, err error) {
628 err = errors.New("missing form body")
631 ct := r.Header.Get("Content-Type")
632 ct, _, err = mime.ParseMediaType(ct)
634 case ct == "application/x-www-form-urlencoded":
635 var reader io.Reader = r.Body
636 maxFormSize := int64(1<<63 - 1)
637 if _, ok := r.Body.(*maxBytesReader); !ok {
638 maxFormSize = int64(10 << 20) // 10 MB is a lot of text.
639 reader = io.LimitReader(r.Body, maxFormSize+1)
641 b, e := ioutil.ReadAll(reader)
648 if int64(len(b)) > maxFormSize {
649 err = errors.New("http: POST too large")
652 vs, e = url.ParseQuery(string(b))
656 case ct == "multipart/form-data":
657 // handled by ParseMultipartForm (which is calling us, or should be)
658 // TODO(bradfitz): there are too many possible
659 // orders to call too many functions here.
660 // Clean this up and write more tests.
661 // request_test.go contains the start of this,
662 // in TestRequestMultipartCallOrder.
667 // ParseForm parses the raw query from the URL and updates r.Form.
669 // For POST or PUT requests, it also parses the request body as a form and
670 // put the results into both r.PostForm and r.Form.
671 // POST and PUT body parameters take precedence over URL query string values
674 // If the request Body's size has not already been limited by MaxBytesReader,
675 // the size is capped at 10MB.
677 // ParseMultipartForm calls ParseForm automatically.
679 func (r *Request) ParseForm() error {
681 if r.PostForm == nil {
682 if r.Method == "POST" || r.Method == "PUT" {
683 r.PostForm, err = parsePostForm(r)
685 if r.PostForm == nil {
686 r.PostForm = make(url.Values)
690 if len(r.PostForm) > 0 {
691 r.Form = make(url.Values)
692 copyValues(r.Form, r.PostForm)
694 var newValues url.Values
697 newValues, e = url.ParseQuery(r.URL.RawQuery)
702 if newValues == nil {
703 newValues = make(url.Values)
708 copyValues(r.Form, newValues)
714 // ParseMultipartForm parses a request body as multipart/form-data.
715 // The whole request body is parsed and up to a total of maxMemory bytes of
716 // its file parts are stored in memory, with the remainder stored on
717 // disk in temporary files.
718 // ParseMultipartForm calls ParseForm if necessary.
719 // After one call to ParseMultipartForm, subsequent calls have no effect.
720 func (r *Request) ParseMultipartForm(maxMemory int64) error {
721 if r.MultipartForm == multipartByReader {
722 return errors.New("http: multipart handled by MultipartReader")
730 if r.MultipartForm != nil {
734 mr, err := r.multipartReader()
735 if err == ErrNotMultipart {
737 } else if err != nil {
741 f, err := mr.ReadForm(maxMemory)
745 for k, v := range f.Value {
746 r.Form[k] = append(r.Form[k], v...)
753 // FormValue returns the first value for the named component of the query.
754 // POST and PUT body parameters take precedence over URL query string values.
755 // FormValue calls ParseMultipartForm and ParseForm if necessary.
756 // To access multiple values of the same key use ParseForm.
757 func (r *Request) FormValue(key string) string {
759 r.ParseMultipartForm(defaultMaxMemory)
761 if vs := r.Form[key]; len(vs) > 0 {
767 // PostFormValue returns the first value for the named component of the POST
768 // or PUT request body. URL query parameters are ignored.
769 // PostFormValue calls ParseMultipartForm and ParseForm if necessary.
770 func (r *Request) PostFormValue(key string) string {
771 if r.PostForm == nil {
772 r.ParseMultipartForm(defaultMaxMemory)
774 if vs := r.PostForm[key]; len(vs) > 0 {
780 // FormFile returns the first file for the provided form key.
781 // FormFile calls ParseMultipartForm and ParseForm if necessary.
782 func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) {
783 if r.MultipartForm == multipartByReader {
784 return nil, nil, errors.New("http: multipart handled by MultipartReader")
786 if r.MultipartForm == nil {
787 err := r.ParseMultipartForm(defaultMaxMemory)
792 if r.MultipartForm != nil && r.MultipartForm.File != nil {
793 if fhs := r.MultipartForm.File[key]; len(fhs) > 0 {
794 f, err := fhs[0].Open()
795 return f, fhs[0], err
798 return nil, nil, ErrMissingFile
801 func (r *Request) expectsContinue() bool {
802 return hasToken(r.Header.get("Expect"), "100-continue")
805 func (r *Request) wantsHttp10KeepAlive() bool {
806 if r.ProtoMajor != 1 || r.ProtoMinor != 0 {
809 return hasToken(r.Header.get("Connection"), "keep-alive")
812 func (r *Request) wantsClose() bool {
813 return hasToken(r.Header.get("Connection"), "close")