1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // HTTP Request reading and parsing.
29 chunkSize = 4 << 10 // 4 KB chunks
30 defaultMaxMemory = 32 << 20 // 32 MB
33 // ErrMissingFile is returned by FormFile when the provided file field name
34 // is either not present in the request or not a file field.
35 var ErrMissingFile = errors.New("http: no such file")
37 // HTTP request parsing errors.
38 type ProtocolError struct {
42 func (err *ProtocolError) Error() string { return err.ErrorString }
45 ErrHeaderTooLong = &ProtocolError{"header too long"}
46 ErrShortBody = &ProtocolError{"entity body too short"}
47 ErrNotSupported = &ProtocolError{"feature not supported"}
48 ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"}
49 ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"}
50 ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"}
51 ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"}
54 type badStringError struct {
59 func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) }
61 // Headers that Request.Write handles itself and should be skipped.
62 var reqWriteExcludeHeader = map[string]bool{
63 "Host": true, // not in Header map anyway
65 "Content-Length": true,
66 "Transfer-Encoding": true,
70 // A Request represents an HTTP request received by a server
71 // or to be sent by a client.
73 Method string // GET, POST, PUT, etc.
75 // URL is created from the URI supplied on the Request-Line
76 // as stored in RequestURI.
78 // For most requests, fields other than Path and RawQuery
79 // will be empty. (See RFC 2616, Section 5.1.2)
82 // The protocol version for incoming requests.
83 // Outgoing requests always use HTTP/1.1.
84 Proto string // "HTTP/1.0"
88 // A header maps request lines to their values.
91 // accept-encoding: gzip, deflate
92 // Accept-Language: en-us
93 // Connection: keep-alive
97 // Header = map[string][]string{
98 // "Accept-Encoding": {"gzip, deflate"},
99 // "Accept-Language": {"en-us"},
100 // "Connection": {"keep-alive"},
103 // HTTP defines that header names are case-insensitive.
104 // The request parser implements this by canonicalizing the
105 // name, making the first character and any characters
106 // following a hyphen uppercase and the rest lowercase.
112 // ContentLength records the length of the associated content.
113 // The value -1 indicates that the length is unknown.
114 // Values >= 0 indicate that the given number of bytes may
115 // be read from Body.
116 // For outgoing requests, a value of 0 means unknown if Body is not nil.
119 // TransferEncoding lists the transfer encodings from outermost to
120 // innermost. An empty list denotes the "identity" encoding.
121 // TransferEncoding can usually be ignored; chunked encoding is
122 // automatically added and removed as necessary when sending and
123 // receiving requests.
124 TransferEncoding []string
126 // Close indicates whether to close the connection after
127 // replying to this request.
130 // The host on which the URL is sought.
131 // Per RFC 2616, this is either the value of the Host: header
132 // or the host name given in the URL itself.
133 // It may be of the form "host:port".
136 // Form contains the parsed form data, including both the URL
137 // field's query parameters and the POST or PUT form data.
138 // This field is only available after ParseForm is called.
139 // The HTTP client ignores Form and uses Body instead.
142 // PostForm contains the parsed form data from POST or PUT
144 // This field is only available after ParseForm is called.
145 // The HTTP client ignores PostForm and uses Body instead.
148 // MultipartForm is the parsed multipart form, including file uploads.
149 // This field is only available after ParseMultipartForm is called.
150 // The HTTP client ignores MultipartForm and uses Body instead.
151 MultipartForm *multipart.Form
153 // Trailer maps trailer keys to values. Like for Header, if the
154 // response has multiple trailer lines with the same key, they will be
155 // concatenated, delimited by commas.
156 // For server requests, Trailer is only populated after Body has been
157 // closed or fully consumed.
158 // Trailer support is only partially complete.
161 // RemoteAddr allows HTTP servers and other software to record
162 // the network address that sent the request, usually for
163 // logging. This field is not filled in by ReadRequest and
164 // has no defined format. The HTTP server in this package
165 // sets RemoteAddr to an "IP:port" address before invoking a
167 // This field is ignored by the HTTP client.
170 // RequestURI is the unmodified Request-URI of the
171 // Request-Line (RFC 2616, Section 5.1) as sent by the client
172 // to a server. Usually the URL field should be used instead.
173 // It is an error to set this field in an HTTP client request.
176 // TLS allows HTTP servers and other software to record
177 // information about the TLS connection on which the request
178 // was received. This field is not filled in by ReadRequest.
179 // The HTTP server in this package sets the field for
180 // TLS-enabled connections before invoking a handler;
181 // otherwise it leaves the field nil.
182 // This field is ignored by the HTTP client.
183 TLS *tls.ConnectionState
186 // ProtoAtLeast returns whether the HTTP protocol used
187 // in the request is at least major.minor.
188 func (r *Request) ProtoAtLeast(major, minor int) bool {
189 return r.ProtoMajor > major ||
190 r.ProtoMajor == major && r.ProtoMinor >= minor
193 // UserAgent returns the client's User-Agent, if sent in the request.
194 func (r *Request) UserAgent() string {
195 return r.Header.Get("User-Agent")
198 // Cookies parses and returns the HTTP cookies sent with the request.
199 func (r *Request) Cookies() []*Cookie {
200 return readCookies(r.Header, "")
203 var ErrNoCookie = errors.New("http: named cookie not present")
205 // Cookie returns the named cookie provided in the request or
206 // ErrNoCookie if not found.
207 func (r *Request) Cookie(name string) (*Cookie, error) {
208 for _, c := range readCookies(r.Header, name) {
211 return nil, ErrNoCookie
214 // AddCookie adds a cookie to the request. Per RFC 6265 section 5.4,
215 // AddCookie does not attach more than one Cookie header field. That
216 // means all cookies, if any, are written into the same line,
217 // separated by semicolon.
218 func (r *Request) AddCookie(c *Cookie) {
219 s := fmt.Sprintf("%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value))
220 if c := r.Header.Get("Cookie"); c != "" {
221 r.Header.Set("Cookie", c+"; "+s)
223 r.Header.Set("Cookie", s)
227 // Referer returns the referring URL, if sent in the request.
229 // Referer is misspelled as in the request itself, a mistake from the
230 // earliest days of HTTP. This value can also be fetched from the
231 // Header map as Header["Referer"]; the benefit of making it available
232 // as a method is that the compiler can diagnose programs that use the
233 // alternate (correct English) spelling req.Referrer() but cannot
234 // diagnose programs that use Header["Referrer"].
235 func (r *Request) Referer() string {
236 return r.Header.Get("Referer")
239 // multipartByReader is a sentinel value.
240 // Its presence in Request.MultipartForm indicates that parsing of the request
241 // body has been handed off to a MultipartReader instead of ParseMultipartFrom.
242 var multipartByReader = &multipart.Form{
243 Value: make(map[string][]string),
244 File: make(map[string][]*multipart.FileHeader),
247 // MultipartReader returns a MIME multipart reader if this is a
248 // multipart/form-data POST request, else returns nil and an error.
249 // Use this function instead of ParseMultipartForm to
250 // process the request body as a stream.
251 func (r *Request) MultipartReader() (*multipart.Reader, error) {
252 if r.MultipartForm == multipartByReader {
253 return nil, errors.New("http: MultipartReader called twice")
255 if r.MultipartForm != nil {
256 return nil, errors.New("http: multipart handled by ParseMultipartForm")
258 r.MultipartForm = multipartByReader
259 return r.multipartReader()
262 func (r *Request) multipartReader() (*multipart.Reader, error) {
263 v := r.Header.Get("Content-Type")
265 return nil, ErrNotMultipart
267 d, params, err := mime.ParseMediaType(v)
268 if err != nil || d != "multipart/form-data" {
269 return nil, ErrNotMultipart
271 boundary, ok := params["boundary"]
273 return nil, ErrMissingBoundary
275 return multipart.NewReader(r.Body, boundary), nil
278 // Return value if nonempty, def otherwise.
279 func valueOrDefault(value, def string) string {
286 const defaultUserAgent = "Go 1.1 package http"
288 // Write writes an HTTP/1.1 request -- header and body -- in wire format.
289 // This method consults the following fields of the request:
292 // Method (defaults to "GET")
298 // If Body is present, Content-Length is <= 0 and TransferEncoding
299 // hasn't been set to "identity", Write adds "Transfer-Encoding:
300 // chunked" to the header. Body is closed after it is sent.
301 func (r *Request) Write(w io.Writer) error {
302 return r.write(w, false, nil)
305 // WriteProxy is like Write but writes the request in the form
306 // expected by an HTTP proxy. In particular, WriteProxy writes the
307 // initial Request-URI line of the request with an absolute URI, per
308 // section 5.1.2 of RFC 2616, including the scheme and host.
309 // In either case, WriteProxy also writes a Host header, using
310 // either r.Host or r.URL.Host.
311 func (r *Request) WriteProxy(w io.Writer) error {
312 return r.write(w, true, nil)
315 // extraHeaders may be nil
316 func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header) error {
320 return errors.New("http: Request.Write on Request with no Host or URL set")
325 ruri := req.URL.RequestURI()
326 if usingProxy && req.URL.Scheme != "" && req.URL.Opaque == "" {
327 ruri = req.URL.Scheme + "://" + host + ruri
328 } else if req.Method == "CONNECT" && req.URL.Path == "" {
329 // CONNECT requests normally give just the host and port, not a full URL.
332 // TODO(bradfitz): escape at least newlines in ruri?
334 // Wrap the writer in a bufio Writer if it's not already buffered.
335 // Don't always call NewWriter, as that forces a bytes.Buffer
336 // and other small bufio Writers to have a minimum 4k buffer
339 if _, ok := w.(io.ByteWriter); !ok {
340 bw = bufio.NewWriter(w)
344 fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri)
347 fmt.Fprintf(w, "Host: %s\r\n", host)
349 // Use the defaultUserAgent unless the Header contains one, which
350 // may be blank to not send the header.
351 userAgent := defaultUserAgent
352 if req.Header != nil {
353 if ua := req.Header["User-Agent"]; len(ua) > 0 {
358 fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent)
361 // Process Body,ContentLength,Close,Trailer
362 tw, err := newTransferWriter(req)
366 err = tw.WriteHeader(w)
371 // TODO: split long values? (If so, should share code with Conn.Write)
372 err = req.Header.WriteSubset(w, reqWriteExcludeHeader)
377 if extraHeaders != nil {
378 err = extraHeaders.Write(w)
384 io.WriteString(w, "\r\n")
386 // Write body and trailer
387 err = tw.WriteBody(w)
398 // ParseHTTPVersion parses a HTTP version string.
399 // "HTTP/1.0" returns (1, 0, true).
400 func ParseHTTPVersion(vers string) (major, minor int, ok bool) {
401 const Big = 1000000 // arbitrary upper bound
408 if !strings.HasPrefix(vers, "HTTP/") {
411 dot := strings.Index(vers, ".")
415 major, err := strconv.Atoi(vers[5:dot])
416 if err != nil || major < 0 || major > Big {
419 minor, err = strconv.Atoi(vers[dot+1:])
420 if err != nil || minor < 0 || minor > Big {
423 return major, minor, true
426 // NewRequest returns a new Request given a method, URL, and optional body.
427 func NewRequest(method, urlStr string, body io.Reader) (*Request, error) {
428 u, err := url.Parse(urlStr)
432 rc, ok := body.(io.ReadCloser)
433 if !ok && body != nil {
434 rc = ioutil.NopCloser(body)
442 Header: make(Header),
447 switch v := body.(type) {
449 req.ContentLength = int64(v.Len())
451 req.ContentLength = int64(v.Len())
452 case *strings.Reader:
453 req.ContentLength = int64(v.Len())
460 // SetBasicAuth sets the request's Authorization header to use HTTP
461 // Basic Authentication with the provided username and password.
463 // With HTTP Basic Authentication the provided username and password
464 // are not encrypted.
465 func (r *Request) SetBasicAuth(username, password string) {
466 s := username + ":" + password
467 r.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(s)))
470 // parseRequestLine parses "GET /foo HTTP/1.1" into its three parts.
471 func parseRequestLine(line string) (method, requestURI, proto string, ok bool) {
472 s1 := strings.Index(line, " ")
473 s2 := strings.Index(line[s1+1:], " ")
474 if s1 < 0 || s2 < 0 {
478 return line[:s1], line[s1+1 : s2], line[s2+1:], true
481 // TODO(bradfitz): use a sync.Cache when available
482 var textprotoReaderCache = make(chan *textproto.Reader, 4)
484 func newTextprotoReader(br *bufio.Reader) *textproto.Reader {
486 case r := <-textprotoReaderCache:
490 return textproto.NewReader(br)
494 func putTextprotoReader(r *textproto.Reader) {
497 case textprotoReaderCache <- r:
502 // ReadRequest reads and parses a request from b.
503 func ReadRequest(b *bufio.Reader) (req *Request, err error) {
505 tp := newTextprotoReader(b)
508 // First line: GET /index.html HTTP/1.0
510 if s, err = tp.ReadLine(); err != nil {
514 putTextprotoReader(tp)
516 err = io.ErrUnexpectedEOF
521 req.Method, req.RequestURI, req.Proto, ok = parseRequestLine(s)
523 return nil, &badStringError{"malformed HTTP request", s}
525 rawurl := req.RequestURI
526 if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok {
527 return nil, &badStringError{"malformed HTTP version", req.Proto}
530 // CONNECT requests are used two different ways, and neither uses a full URL:
531 // The standard use is to tunnel HTTPS through an HTTP proxy.
532 // It looks like "CONNECT www.google.com:443 HTTP/1.1", and the parameter is
533 // just the authority section of a URL. This information should go in req.URL.Host.
535 // The net/rpc package also uses CONNECT, but there the parameter is a path
536 // that starts with a slash. It can be parsed with the regular URL parser,
537 // and the path will end up in req.URL.Path, where it needs to be in order for
539 justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/")
541 rawurl = "http://" + rawurl
544 if req.URL, err = url.ParseRequestURI(rawurl); err != nil {
549 // Strip the bogus "http://" back off.
553 // Subsequent lines: Key: value.
554 mimeHeader, err := tp.ReadMIMEHeader()
558 req.Header = Header(mimeHeader)
560 // RFC2616: Must treat
561 // GET /index.html HTTP/1.1
562 // Host: www.google.com
564 // GET http://www.google.com/index.html HTTP/1.1
565 // Host: doesntmatter
566 // the same. In the second case, any Host line is ignored.
567 req.Host = req.URL.Host
569 req.Host = req.Header.get("Host")
571 delete(req.Header, "Host")
573 fixPragmaCacheControl(req.Header)
575 // TODO: Parse specific header values:
589 // If-Unmodified-Since
591 // Proxy-Authorization
593 // TE (transfer-codings)
601 err = readTransfer(req, b)
609 // MaxBytesReader is similar to io.LimitReader but is intended for
610 // limiting the size of incoming request bodies. In contrast to
611 // io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a
612 // non-EOF error for a Read beyond the limit, and Closes the
613 // underlying reader when its Close method is called.
615 // MaxBytesReader prevents clients from accidentally or maliciously
616 // sending a large request and wasting server resources.
617 func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser {
618 return &maxBytesReader{w: w, r: r, n: n}
621 type maxBytesReader struct {
623 r io.ReadCloser // underlying reader
624 n int64 // max bytes remaining
628 func (l *maxBytesReader) Read(p []byte) (n int, err error) {
632 if res, ok := l.w.(*response); ok {
633 res.requestTooLarge()
636 return 0, errors.New("http: request body too large")
638 if int64(len(p)) > l.n {
646 func (l *maxBytesReader) Close() error {
650 func copyValues(dst, src url.Values) {
651 for k, vs := range src {
652 for _, value := range vs {
658 func parsePostForm(r *Request) (vs url.Values, err error) {
660 err = errors.New("missing form body")
663 ct := r.Header.Get("Content-Type")
664 ct, _, err = mime.ParseMediaType(ct)
666 case ct == "application/x-www-form-urlencoded":
667 var reader io.Reader = r.Body
668 maxFormSize := int64(1<<63 - 1)
669 if _, ok := r.Body.(*maxBytesReader); !ok {
670 maxFormSize = int64(10 << 20) // 10 MB is a lot of text.
671 reader = io.LimitReader(r.Body, maxFormSize+1)
673 b, e := ioutil.ReadAll(reader)
680 if int64(len(b)) > maxFormSize {
681 err = errors.New("http: POST too large")
684 vs, e = url.ParseQuery(string(b))
688 case ct == "multipart/form-data":
689 // handled by ParseMultipartForm (which is calling us, or should be)
690 // TODO(bradfitz): there are too many possible
691 // orders to call too many functions here.
692 // Clean this up and write more tests.
693 // request_test.go contains the start of this,
694 // in TestRequestMultipartCallOrder.
699 // ParseForm parses the raw query from the URL and updates r.Form.
701 // For POST or PUT requests, it also parses the request body as a form and
702 // put the results into both r.PostForm and r.Form.
703 // POST and PUT body parameters take precedence over URL query string values
706 // If the request Body's size has not already been limited by MaxBytesReader,
707 // the size is capped at 10MB.
709 // ParseMultipartForm calls ParseForm automatically.
711 func (r *Request) ParseForm() error {
713 if r.PostForm == nil {
714 if r.Method == "POST" || r.Method == "PUT" {
715 r.PostForm, err = parsePostForm(r)
717 if r.PostForm == nil {
718 r.PostForm = make(url.Values)
722 if len(r.PostForm) > 0 {
723 r.Form = make(url.Values)
724 copyValues(r.Form, r.PostForm)
726 var newValues url.Values
729 newValues, e = url.ParseQuery(r.URL.RawQuery)
734 if newValues == nil {
735 newValues = make(url.Values)
740 copyValues(r.Form, newValues)
746 // ParseMultipartForm parses a request body as multipart/form-data.
747 // The whole request body is parsed and up to a total of maxMemory bytes of
748 // its file parts are stored in memory, with the remainder stored on
749 // disk in temporary files.
750 // ParseMultipartForm calls ParseForm if necessary.
751 // After one call to ParseMultipartForm, subsequent calls have no effect.
752 func (r *Request) ParseMultipartForm(maxMemory int64) error {
753 if r.MultipartForm == multipartByReader {
754 return errors.New("http: multipart handled by MultipartReader")
762 if r.MultipartForm != nil {
766 mr, err := r.multipartReader()
767 if err == ErrNotMultipart {
769 } else if err != nil {
773 f, err := mr.ReadForm(maxMemory)
777 for k, v := range f.Value {
778 r.Form[k] = append(r.Form[k], v...)
785 // FormValue returns the first value for the named component of the query.
786 // POST and PUT body parameters take precedence over URL query string values.
787 // FormValue calls ParseMultipartForm and ParseForm if necessary.
788 // To access multiple values of the same key use ParseForm.
789 func (r *Request) FormValue(key string) string {
791 r.ParseMultipartForm(defaultMaxMemory)
793 if vs := r.Form[key]; len(vs) > 0 {
799 // PostFormValue returns the first value for the named component of the POST
800 // or PUT request body. URL query parameters are ignored.
801 // PostFormValue calls ParseMultipartForm and ParseForm if necessary.
802 func (r *Request) PostFormValue(key string) string {
803 if r.PostForm == nil {
804 r.ParseMultipartForm(defaultMaxMemory)
806 if vs := r.PostForm[key]; len(vs) > 0 {
812 // FormFile returns the first file for the provided form key.
813 // FormFile calls ParseMultipartForm and ParseForm if necessary.
814 func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) {
815 if r.MultipartForm == multipartByReader {
816 return nil, nil, errors.New("http: multipart handled by MultipartReader")
818 if r.MultipartForm == nil {
819 err := r.ParseMultipartForm(defaultMaxMemory)
824 if r.MultipartForm != nil && r.MultipartForm.File != nil {
825 if fhs := r.MultipartForm.File[key]; len(fhs) > 0 {
826 f, err := fhs[0].Open()
827 return f, fhs[0], err
830 return nil, nil, ErrMissingFile
833 func (r *Request) expectsContinue() bool {
834 return hasToken(r.Header.get("Expect"), "100-continue")
837 func (r *Request) wantsHttp10KeepAlive() bool {
838 if r.ProtoMajor != 1 || r.ProtoMinor != 0 {
841 return hasToken(r.Header.get("Connection"), "keep-alive")
844 func (r *Request) wantsClose() bool {
845 return hasToken(r.Header.get("Connection"), "close")