1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // HTTP Request reading and parsing.
7 // Package http implements parsing of HTTP requests, replies, and URLs and
8 // provides an extensible HTTP server and a basic HTTP client.
27 maxLineLength = 4096 // assumed <= bufio.defaultBufSize
30 chunkSize = 4 << 10 // 4 KB chunks
31 defaultMaxMemory = 32 << 20 // 32 MB
34 // ErrMissingFile is returned by FormFile when the provided file field name
35 // is either not present in the request or not a file field.
36 var ErrMissingFile = os.ErrorString("http: no such file")
38 // HTTP request parsing errors.
39 type ProtocolError struct {
44 ErrLineTooLong = &ProtocolError{"header line too long"}
45 ErrHeaderTooLong = &ProtocolError{"header too long"}
46 ErrShortBody = &ProtocolError{"entity body too short"}
47 ErrNotSupported = &ProtocolError{"feature not supported"}
48 ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"}
49 ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"}
50 ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"}
51 ErrMissingBoundary = &ProtocolError{"no multipart boundary param Content-Type"}
54 type badStringError struct {
59 func (e *badStringError) String() string { return fmt.Sprintf("%s %q", e.what, e.str) }
61 var reqExcludeHeader = map[string]bool{
65 "Content-Length": true,
66 "Transfer-Encoding": true,
70 // A Request represents a parsed HTTP request header.
72 Method string // GET, POST, PUT, etc.
73 RawURL string // The raw URL given in the request.
74 URL *URL // Parsed URL.
76 // The protocol version for incoming requests.
77 // Outgoing requests always use HTTP/1.1.
78 Proto string // "HTTP/1.0"
82 // A header maps request lines to their values.
85 // accept-encoding: gzip, deflate
86 // Accept-Language: en-us
87 // Connection: keep-alive
91 // Header = map[string]string{
92 // "Accept-Encoding": "gzip, deflate",
93 // "Accept-Language": "en-us",
94 // "Connection": "keep-alive",
97 // HTTP defines that header names are case-insensitive.
98 // The request parser implements this by canonicalizing the
99 // name, making the first character and any characters
100 // following a hyphen uppercase and the rest lowercase.
103 // Cookie records the HTTP cookies sent with the request.
109 // ContentLength records the length of the associated content.
110 // The value -1 indicates that the length is unknown.
111 // Values >= 0 indicate that the given number of bytes may be read from Body.
114 // TransferEncoding lists the transfer encodings from outermost to innermost.
115 // An empty list denotes the "identity" encoding.
116 TransferEncoding []string
118 // Whether to close the connection after replying to this request.
121 // The host on which the URL is sought.
122 // Per RFC 2616, this is either the value of the Host: header
123 // or the host name given in the URL itself.
126 // The referring URL, if sent in the request.
128 // Referer is misspelled as in the request itself,
129 // a mistake from the earliest days of HTTP.
130 // This value can also be fetched from the Header map
131 // as Header["Referer"]; the benefit of making it
132 // available as a structure field is that the compiler
133 // can diagnose programs that use the alternate
134 // (correct English) spelling req.Referrer but cannot
135 // diagnose programs that use Header["Referrer"].
138 // The User-Agent: header string, if sent in the request.
141 // The parsed form. Only available after ParseForm is called.
142 Form map[string][]string
144 // The parsed multipart form, including file uploads.
145 // Only available after ParseMultipartForm is called.
146 MultipartForm *multipart.Form
148 // Trailer maps trailer keys to values. Like for Header, if the
149 // response has multiple trailer lines with the same key, they will be
150 // concatenated, delimited by commas.
153 // RemoteAddr allows HTTP servers and other software to record
154 // the network address that sent the request, usually for
155 // logging. This field is not filled in by ReadRequest and
156 // has no defined format. The HTTP server in this package
157 // sets RemoteAddr to an "IP:port" address before invoking a
161 // TLS allows HTTP servers and other software to record
162 // information about the TLS connection on which the request
163 // was received. This field is not filled in by ReadRequest.
164 // The HTTP server in this package sets the field for
165 // TLS-enabled connections before invoking a handler;
166 // otherwise it leaves the field nil.
167 TLS *tls.ConnectionState
170 // ProtoAtLeast returns whether the HTTP protocol used
171 // in the request is at least major.minor.
172 func (r *Request) ProtoAtLeast(major, minor int) bool {
173 return r.ProtoMajor > major ||
174 r.ProtoMajor == major && r.ProtoMinor >= minor
177 // multipartByReader is a sentinel value.
178 // Its presence in Request.MultipartForm indicates that parsing of the request
179 // body has been handed off to a MultipartReader instead of ParseMultipartFrom.
180 var multipartByReader = &multipart.Form{
181 Value: make(map[string][]string),
182 File: make(map[string][]*multipart.FileHeader),
185 // MultipartReader returns a MIME multipart reader if this is a
186 // multipart/form-data POST request, else returns nil and an error.
187 // Use this function instead of ParseMultipartForm to
188 // process the request body as a stream.
189 func (r *Request) MultipartReader() (multipart.Reader, os.Error) {
190 if r.MultipartForm == multipartByReader {
191 return nil, os.NewError("http: MultipartReader called twice")
193 if r.MultipartForm != nil {
194 return nil, os.NewError("http: multipart handled by ParseMultipartForm")
196 r.MultipartForm = multipartByReader
197 return r.multipartReader()
200 func (r *Request) multipartReader() (multipart.Reader, os.Error) {
201 v := r.Header.Get("Content-Type")
203 return nil, ErrNotMultipart
205 d, params := mime.ParseMediaType(v)
206 if d != "multipart/form-data" {
207 return nil, ErrNotMultipart
209 boundary, ok := params["boundary"]
211 return nil, ErrMissingBoundary
213 return multipart.NewReader(r.Body, boundary), nil
216 // Return value if nonempty, def otherwise.
217 func valueOrDefault(value, def string) string {
224 const defaultUserAgent = "Go http package"
226 // Write writes an HTTP/1.1 request -- header and body -- in wire format.
227 // This method consults the following fields of req:
229 // RawURL, if non-empty, or else URL
230 // Method (defaults to "GET")
231 // UserAgent (defaults to defaultUserAgent)
239 // If Body is present but Content-Length is <= 0, Write adds
240 // "Transfer-Encoding: chunked" to the header. Body is closed after
242 func (req *Request) Write(w io.Writer) os.Error {
243 return req.write(w, false)
246 // WriteProxy is like Write but writes the request in the form
247 // expected by an HTTP proxy. It includes the scheme and host
248 // name in the URI instead of using a separate Host: header line.
249 // If req.RawURL is non-empty, WriteProxy uses it unchanged
250 // instead of URL but still omits the Host: header.
251 func (req *Request) WriteProxy(w io.Writer) os.Error {
252 return req.write(w, true)
255 func (req *Request) write(w io.Writer, usingProxy bool) os.Error {
263 uri = valueOrDefault(urlEscape(req.URL.Path, encodePath), "/")
264 if req.URL.RawQuery != "" {
265 uri += "?" + req.URL.RawQuery
268 if uri == "" || uri[0] != '/' {
271 uri = req.URL.Scheme + "://" + host + uri
275 fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), uri)
279 fmt.Fprintf(w, "Host: %s\r\n", host)
281 fmt.Fprintf(w, "User-Agent: %s\r\n", valueOrDefault(req.UserAgent, defaultUserAgent))
282 if req.Referer != "" {
283 fmt.Fprintf(w, "Referer: %s\r\n", req.Referer)
286 // Process Body,ContentLength,Close,Trailer
287 tw, err := newTransferWriter(req)
291 err = tw.WriteHeader(w)
296 // TODO: split long values? (If so, should share code with Conn.Write)
297 // TODO: if Header includes values for Host, User-Agent, or Referer, this
298 // may conflict with the User-Agent or Referer headers we add manually.
299 // One solution would be to remove the Host, UserAgent, and Referer fields
300 // from Request, and introduce Request methods along the lines of
301 // Response.{GetHeader,AddHeader} and string constants for "Host",
302 // "User-Agent" and "Referer".
303 err = req.Header.WriteSubset(w, reqExcludeHeader)
308 if err = writeCookies(w, req.Cookie); err != nil {
312 io.WriteString(w, "\r\n")
314 // Write body and trailer
315 err = tw.WriteBody(w)
323 // Read a line of bytes (up to \n) from b.
324 // Give up if the line exceeds maxLineLength.
325 // The returned bytes are a pointer into storage in
326 // the bufio, so they are only valid until the next bufio read.
327 func readLineBytes(b *bufio.Reader) (p []byte, err os.Error) {
328 if p, err = b.ReadSlice('\n'); err != nil {
329 // We always know when EOF is coming.
330 // If the caller asked for a line, there should be a line.
332 err = io.ErrUnexpectedEOF
333 } else if err == bufio.ErrBufferFull {
338 if len(p) >= maxLineLength {
339 return nil, ErrLineTooLong
342 // Chop off trailing white space.
344 for i = len(p); i > 0; i-- {
345 if c := p[i-1]; c != ' ' && c != '\r' && c != '\t' && c != '\n' {
352 // readLineBytes, but convert the bytes into a string.
353 func readLine(b *bufio.Reader) (s string, err os.Error) {
354 p, e := readLineBytes(b)
358 return string(p), nil
361 // Convert decimal at s[i:len(s)] to integer,
362 // returning value, string position where the digits stopped,
363 // and whether there was a valid number (digits, not too big).
364 func atoi(s string, i int) (n, i1 int, ok bool) {
366 if i >= len(s) || s[i] < '0' || s[i] > '9' {
370 for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ {
371 n = n*10 + int(s[i]-'0')
379 // ParseHTTPVersion parses a HTTP version string.
380 // "HTTP/1.0" returns (1, 0, true).
381 func ParseHTTPVersion(vers string) (major, minor int, ok bool) {
382 if len(vers) < 5 || vers[0:5] != "HTTP/" {
385 major, i, ok := atoi(vers, 5)
386 if !ok || i >= len(vers) || vers[i] != '.' {
389 minor, i, ok = atoi(vers, i+1)
390 if !ok || i != len(vers) {
393 return major, minor, true
396 type chunkedReader struct {
398 n uint64 // unread bytes in chunk
402 func newChunkedReader(r *bufio.Reader) *chunkedReader {
403 return &chunkedReader{r: r}
406 func (cr *chunkedReader) beginChunk() {
409 line, cr.err = readLine(cr.r)
413 cr.n, cr.err = strconv.Btoui64(line, 16)
420 line, cr.err = readLine(cr.r)
432 func (cr *chunkedReader) Read(b []uint8) (n int, err os.Error) {
442 if uint64(len(b)) > cr.n {
445 n, cr.err = cr.r.Read(b)
447 if cr.n == 0 && cr.err == nil {
448 // end of chunk (CRLF)
450 if _, cr.err = io.ReadFull(cr.r, b); cr.err == nil {
451 if b[0] != '\r' || b[1] != '\n' {
452 cr.err = os.NewError("malformed chunked encoding")
459 // NewRequest returns a new Request given a method, URL, and optional body.
460 func NewRequest(method, url string, body io.Reader) (*Request, os.Error) {
461 u, err := ParseURL(url)
465 rc, ok := body.(io.ReadCloser)
466 if !ok && body != nil {
467 rc = ioutil.NopCloser(body)
475 Header: make(Header),
482 // ReadRequest reads and parses a request from b.
483 func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) {
485 tp := textproto.NewReader(b)
488 // First line: GET /index.html HTTP/1.0
490 if s, err = tp.ReadLine(); err != nil {
492 err = io.ErrUnexpectedEOF
498 if f = strings.Split(s, " ", 3); len(f) < 3 {
499 return nil, &badStringError{"malformed HTTP request", s}
501 req.Method, req.RawURL, req.Proto = f[0], f[1], f[2]
503 if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok {
504 return nil, &badStringError{"malformed HTTP version", req.Proto}
507 if req.URL, err = ParseRequestURL(req.RawURL); err != nil {
511 // Subsequent lines: Key: value.
512 mimeHeader, err := tp.ReadMIMEHeader()
516 req.Header = Header(mimeHeader)
518 // RFC2616: Must treat
519 // GET /index.html HTTP/1.1
520 // Host: www.google.com
522 // GET http://www.google.com/index.html HTTP/1.1
523 // Host: doesntmatter
524 // the same. In the second case, any Host line is ignored.
525 req.Host = req.URL.Host
527 req.Host = req.Header.Get("Host")
529 req.Header.Del("Host")
531 fixPragmaCacheControl(req.Header)
533 // Pull out useful fields as a convenience to clients.
534 req.Referer = req.Header.Get("Referer")
535 req.Header.Del("Referer")
537 req.UserAgent = req.Header.Get("User-Agent")
538 req.Header.Del("User-Agent")
540 // TODO: Parse specific header values:
554 // If-Unmodified-Since
556 // Proxy-Authorization
558 // TE (transfer-codings)
566 err = readTransfer(req, b)
571 req.Cookie = readCookies(req.Header)
576 // ParseQuery parses the URL-encoded query string and returns
577 // a map listing the values specified for each key.
578 // ParseQuery always returns a non-nil map containing all the
579 // valid query parameters found; err describes the first decoding error
580 // encountered, if any.
581 func ParseQuery(query string) (m map[string][]string, err os.Error) {
582 m = make(map[string][]string)
583 err = parseQuery(m, query)
587 func parseQuery(m map[string][]string, query string) (err os.Error) {
588 for _, kv := range strings.Split(query, "&", -1) {
592 kvPair := strings.Split(kv, "=", 2)
594 var key, value string
596 key, e = URLUnescape(kvPair[0])
597 if e == nil && len(kvPair) > 1 {
598 value, e = URLUnescape(kvPair[1])
604 vec := vector.StringVector(m[key])
611 // ParseForm parses the raw query.
612 // For POST requests, it also parses the request body as a form.
613 // ParseMultipartForm calls ParseForm automatically.
615 func (r *Request) ParseForm() (err os.Error) {
620 r.Form = make(map[string][]string)
622 err = parseQuery(r.Form, r.URL.RawQuery)
624 if r.Method == "POST" {
626 return os.ErrorString("missing form body")
628 ct := r.Header.Get("Content-Type")
629 switch strings.Split(ct, ";", 2)[0] {
630 case "text/plain", "application/x-www-form-urlencoded", "":
631 const maxFormSize = int64(10 << 20) // 10 MB is a lot of text.
632 b, e := ioutil.ReadAll(io.LimitReader(r.Body, maxFormSize+1))
639 if int64(len(b)) > maxFormSize {
640 return os.NewError("http: POST too large")
642 e = parseQuery(r.Form, string(b))
646 case "multipart/form-data":
647 // handled by ParseMultipartForm
649 return &badStringError{"unknown Content-Type", ct}
655 // ParseMultipartForm parses a request body as multipart/form-data.
656 // The whole request body is parsed and up to a total of maxMemory bytes of
657 // its file parts are stored in memory, with the remainder stored on
658 // disk in temporary files.
659 // ParseMultipartForm calls ParseForm if necessary.
660 // After one call to ParseMultipartForm, subsequent calls have no effect.
661 func (r *Request) ParseMultipartForm(maxMemory int64) os.Error {
668 if r.MultipartForm != nil {
671 if r.MultipartForm == multipartByReader {
672 return os.NewError("http: multipart handled by MultipartReader")
675 mr, err := r.multipartReader()
676 if err == ErrNotMultipart {
678 } else if err != nil {
682 f, err := mr.ReadForm(maxMemory)
686 for k, v := range f.Value {
687 r.Form[k] = append(r.Form[k], v...)
694 // FormValue returns the first value for the named component of the query.
695 // FormValue calls ParseMultipartForm and ParseForm if necessary.
696 func (r *Request) FormValue(key string) string {
698 r.ParseMultipartForm(defaultMaxMemory)
700 if vs := r.Form[key]; len(vs) > 0 {
706 // FormFile returns the first file for the provided form key.
707 // FormFile calls ParseMultipartForm and ParseForm if necessary.
708 func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, os.Error) {
709 if r.MultipartForm == multipartByReader {
710 return nil, nil, os.NewError("http: multipart handled by MultipartReader")
712 if r.MultipartForm == nil {
713 err := r.ParseMultipartForm(defaultMaxMemory)
718 if r.MultipartForm != nil && r.MultipartForm.File != nil {
719 if fhs := r.MultipartForm.File[key]; len(fhs) > 0 {
720 f, err := fhs[0].Open()
721 return f, fhs[0], err
724 return nil, nil, ErrMissingFile
727 func (r *Request) expectsContinue() bool {
728 return strings.ToLower(r.Header.Get("Expect")) == "100-continue"
731 func (r *Request) wantsHttp10KeepAlive() bool {
732 if r.ProtoMajor != 1 || r.ProtoMinor != 0 {
735 return strings.Contains(strings.ToLower(r.Header.Get("Connection")), "keep-alive")