From: ian Date: Thu, 18 Feb 2016 05:56:46 +0000 (+0000) Subject: libgo: Update to final Go 1.6 release. X-Git-Tag: upstream/6.1~1019 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=39d861e813784a735a38cfcb3fc205066007c675;p=platform%2Fupstream%2Flinaro-gcc.git libgo: Update to final Go 1.6 release. Reviewed-on: https://go-review.googlesource.com/19592 git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@233515 138bc75d-0d04-0410-961f-82ee72b054a4 --- diff --git a/gcc/go/gofrontend/MERGE b/gcc/go/gofrontend/MERGE index 228dfc1..038e434 100644 --- a/gcc/go/gofrontend/MERGE +++ b/gcc/go/gofrontend/MERGE @@ -1,4 +1,4 @@ -22278c6e8ce3982b09111183bc6addf0184bef1f +1c3747d20789c73447ff71cbc739f7423c4bdf67 The first line of this file holds the git revision number of the last merge done from the gofrontend repository. diff --git a/libgo/MERGE b/libgo/MERGE index 0522f32..3cb26c3 100644 --- a/libgo/MERGE +++ b/libgo/MERGE @@ -1,4 +1,4 @@ -036b8fd40b60830ca1d152f17148e52b96d8aa6c +7bc40ffb05d8813bf9b41a331b45d37216f9e747 The first line of this file holds the git revision number of the last merge done from the master library sources. diff --git a/libgo/VERSION b/libgo/VERSION index c9b2fc0..0be0043 100644 --- a/libgo/VERSION +++ b/libgo/VERSION @@ -1 +1 @@ -go1.6rc1 \ No newline at end of file +go1.6 \ No newline at end of file diff --git a/libgo/go/archive/zip/reader.go b/libgo/go/archive/zip/reader.go index 84a9d41..10e8172 100644 --- a/libgo/go/archive/zip/reader.go +++ b/libgo/go/archive/zip/reader.go @@ -330,7 +330,17 @@ func readDirectoryHeader(f *File, r io.Reader) error { } } - if needUSize || needCSize || needHeaderOffset { + // Assume that uncompressed size 2³²-1 could plausibly happen in + // an old zip32 file that was sharding inputs into the largest chunks + // possible (or is just malicious; search the web for 42.zip). + // If needUSize is true still, it means we didn't see a zip64 extension. + // As long as the compressed size is not also 2³²-1 (implausible) + // and the header is not also 2³²-1 (equally implausible), + // accept the uncompressed size 2³²-1 as valid. + // If nothing else, this keeps archive/zip working with 42.zip. + _ = needUSize + + if needCSize || needHeaderOffset { return ErrFormat } diff --git a/libgo/go/archive/zip/reader_test.go b/libgo/go/archive/zip/reader_test.go index 8f7e8bf..72cf5d9 100644 --- a/libgo/go/archive/zip/reader_test.go +++ b/libgo/go/archive/zip/reader_test.go @@ -27,12 +27,24 @@ type ZipTest struct { } type ZipTestFile struct { - Name string - Content []byte // if blank, will attempt to compare against File + Name string + Mode os.FileMode + Mtime string // optional, modified time in format "mm-dd-yy hh:mm:ss" + + // Information describing expected zip file content. + // First, reading the entire content should produce the error ContentErr. + // Second, if ContentErr==nil, the content should match Content. + // If content is large, an alternative to setting Content is to set File, + // which names a file in the testdata/ directory containing the + // uncompressed expected content. + // If content is very large, an alternative to setting Content or File + // is to set Size, which will then be checked against the header-reported size + // but will bypass the decompressing of the actual data. + // This last option is used for testing very large (multi-GB) compressed files. ContentErr error - File string // name of file to compare to (relative to testdata/) - Mtime string // modified time in format "mm-dd-yy hh:mm:ss" - Mode os.FileMode + Content []byte + File string + Size uint64 } // Caution: The Mtime values found for the test files should correspond to @@ -248,6 +260,19 @@ var tests = []ZipTest{ }, }, }, + // Largest possible non-zip64 file, with no zip64 header. + { + Name: "big.zip", + Source: returnBigZipBytes, + File: []ZipTestFile{ + { + Name: "big.file", + Content: nil, + Size: 1<<32 - 1, + Mode: 0666, + }, + }, + }, } var crossPlatform = []ZipTestFile{ @@ -356,13 +381,31 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) { testFileMode(t, zt.Name, f, ft.Mode) - var b bytes.Buffer + size := uint64(f.UncompressedSize) + if size == uint32max { + size = f.UncompressedSize64 + } else if size != f.UncompressedSize64 { + t.Errorf("%v: UncompressedSize=%#x does not match UncompressedSize64=%#x", f.Name, size, f.UncompressedSize64) + } + r, err := f.Open() if err != nil { t.Errorf("%s: %v", zt.Name, err) return } + // For very large files, just check that the size is correct. + // The content is expected to be all zeros. + // Don't bother uncompressing: too big. + if ft.Content == nil && ft.File == "" && ft.Size > 0 { + if size != ft.Size { + t.Errorf("%v: uncompressed size %#x, want %#x", size, ft.Size) + } + r.Close() + return + } + + var b bytes.Buffer _, err = io.Copy(&b, r) if err != ft.ContentErr { t.Errorf("%s: copying contents: %v (want %v)", zt.Name, err, ft.ContentErr) @@ -372,10 +415,6 @@ func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) { } r.Close() - size := uint64(f.UncompressedSize) - if size == uint32max { - size = f.UncompressedSize64 - } if g := uint64(b.Len()); g != size { t.Errorf("%v: read %v bytes but f.UncompressedSize == %v", f.Name, g, size) } @@ -510,6 +549,182 @@ func returnRecursiveZip() (r io.ReaderAt, size int64) { return bytes.NewReader(b), int64(len(b)) } +// biggestZipBytes returns the bytes of a zip file biggest.zip +// that contains a zip file bigger.zip that contains a zip file +// big.zip that contains big.file, which contains 2³²-1 zeros. +// The big.zip file is interesting because it has no zip64 header, +// much like the innermost zip files in the well-known 42.zip. +// +// biggest.zip was generated by changing isZip64 to use > uint32max +// instead of >= uint32max and then running this program: +// +// package main +// +// import ( +// "archive/zip" +// "bytes" +// "io" +// "io/ioutil" +// "log" +// ) +// +// type zeros struct{} +// +// func (zeros) Read(b []byte) (int, error) { +// for i := range b { +// b[i] = 0 +// } +// return len(b), nil +// } +// +// func main() { +// bigZip := makeZip("big.file", io.LimitReader(zeros{}, 1<<32-1)) +// if err := ioutil.WriteFile("/tmp/big.zip", bigZip, 0666); err != nil { +// log.Fatal(err) +// } +// +// biggerZip := makeZip("big.zip", bytes.NewReader(bigZip)) +// if err := ioutil.WriteFile("/tmp/bigger.zip", biggerZip, 0666); err != nil { +// log.Fatal(err) +// } +// +// biggestZip := makeZip("bigger.zip", bytes.NewReader(biggerZip)) +// if err := ioutil.WriteFile("/tmp/biggest.zip", biggestZip, 0666); err != nil { +// log.Fatal(err) +// } +// } +// +// func makeZip(name string, r io.Reader) []byte { +// var buf bytes.Buffer +// w := zip.NewWriter(&buf) +// wf, err := w.Create(name) +// if err != nil { +// log.Fatal(err) +// } +// if _, err = io.Copy(wf, r); err != nil { +// log.Fatal(err) +// } +// if err := w.Close(); err != nil { +// log.Fatal(err) +// } +// return buf.Bytes() +// } +// +// The 4 GB of zeros compresses to 4 MB, which compresses to 20 kB, +// which compresses to 1252 bytes (in the hex dump below). +// +// It's here in hex for the same reason as rZipBytes above: to avoid +// problems with on-disk virus scanners or other zip processors. +// +func biggestZipBytes() []byte { + s := ` +0000000 50 4b 03 04 14 00 08 00 08 00 00 00 00 00 00 00 +0000010 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 62 69 +0000020 67 67 65 72 2e 7a 69 70 ec dc 6b 4c 53 67 18 07 +0000030 f0 16 c5 ca 65 2e cb b8 94 20 61 1f 44 33 c7 cd +0000040 c0 86 4a b5 c0 62 8a 61 05 c6 cd 91 b2 54 8c 1b +0000050 63 8b 03 9c 1b 95 52 5a e3 a0 19 6c b2 05 59 44 +0000060 64 9d 73 83 71 11 46 61 14 b9 1d 14 09 4a c3 60 +0000070 2e 4c 6e a5 60 45 02 62 81 95 b6 94 9e 9e 77 e7 +0000080 d0 43 b6 f8 71 df 96 3c e7 a4 69 ce bf cf e9 79 +0000090 ce ef 79 3f bf f1 31 db b6 bb 31 76 92 e7 f3 07 +00000a0 8b fc 9c ca cc 08 cc cb cc 5e d2 1c 88 d9 7e bb +00000b0 4f bb 3a 3f 75 f1 5d 7f 8f c2 68 67 77 8f 25 ff +00000c0 84 e2 93 2d ef a4 95 3d 71 4e 2c b9 b0 87 c3 be +00000d0 3d f8 a7 60 24 61 c5 ef ae 9e c8 6c 6d 4e 69 c8 +00000e0 67 65 34 f8 37 76 2d 76 5c 54 f3 95 65 49 c7 0f +00000f0 18 71 4b 7e 5b 6a d1 79 47 61 41 b0 4e 2a 74 45 +0000100 43 58 12 b2 5a a5 c6 7d 68 55 88 d4 98 75 18 6d +0000110 08 d1 1f 8f 5a 9e 96 ee 45 cf a4 84 4e 4b e8 50 +0000120 a7 13 d9 06 de 52 81 97 36 b2 d7 b8 fc 2b 5f 55 +0000130 23 1f 32 59 cf 30 27 fb e2 8a b9 de 45 dd 63 9c +0000140 4b b5 8b 96 4c 7a 62 62 cc a1 a7 cf fa f1 fe dd +0000150 54 62 11 bf 36 78 b3 c7 b1 b5 f2 61 4d 4e dd 66 +0000160 32 2e e6 70 34 5f f4 c9 e6 6c 43 6f da 6b c6 c3 +0000170 09 2c ce 09 57 7f d2 7e b4 23 ba 7c 1b 99 bc 22 +0000180 3e f1 de 91 2f e3 9c 1b 82 cc c2 84 39 aa e6 de +0000190 b4 69 fc cc cb 72 a6 61 45 f0 d3 1d 26 19 7c 8d +00001a0 29 c8 66 02 be 77 6a f9 3d 34 79 17 19 c8 96 24 +00001b0 a3 ac e4 dd 3b 1a 8e c6 fe 96 38 6b bf 67 5a 23 +00001c0 f4 16 f4 e6 8a b4 fc c2 cd bf 95 66 1d bb 35 aa +00001d0 92 7d 66 d8 08 8d a5 1f 54 2a af 09 cf 61 ff d2 +00001e0 85 9d 8f b6 d7 88 07 4a 86 03 db 64 f3 d9 92 73 +00001f0 df ec a7 fc 23 4c 8d 83 79 63 2a d9 fd 8d b3 c8 +0000200 8f 7e d4 19 85 e6 8d 1c 76 f0 8b 58 32 fd 9a d6 +0000210 85 e2 48 ad c3 d5 60 6f 7e 22 dd ef 09 49 7c 7f +0000220 3a 45 c3 71 b7 df f3 4c 63 fb b5 d9 31 5f 6e d6 +0000230 24 1d a4 4a fe 32 a7 5c 16 48 5c 3e 08 6b 8a d3 +0000240 25 1d a2 12 a5 59 24 ea 20 5f 52 6d ad 94 db 6b +0000250 94 b9 5d eb 4b a7 5c 44 bb 1e f2 3c 6b cf 52 c9 +0000260 e9 e5 ba 06 b9 c4 e5 0a d0 00 0d d0 00 0d d0 00 +0000270 0d d0 00 0d d0 00 0d d0 00 0d d0 00 0d d0 00 0d +0000280 d0 00 0d d0 00 0d d0 00 0d d0 00 0d d0 00 0d d0 +0000290 00 0d d0 00 0d d0 00 0d d0 00 0d d0 00 0d d0 00 +00002a0 0d d0 00 cd ff 9e 46 86 fa a7 7d 3a 43 d7 8e 10 +00002b0 52 e9 be e6 6e cf eb 9e 85 4d 65 ce cc 30 c1 44 +00002c0 c0 4e af bc 9c 6c 4b a0 d7 54 ff 1d d5 5c 89 fb +00002d0 b5 34 7e c4 c2 9e f5 a0 f6 5b 7e 6e ca 73 c7 ef +00002e0 5d be de f9 e8 81 eb a5 0a a5 63 54 2c d7 1c d1 +00002f0 89 17 85 f8 16 94 f2 8a b2 a3 f5 b6 6d df 75 cd +0000300 90 dd 64 bd 5d 55 4e f2 55 19 1b b7 cc ef 1b ea +0000310 2e 05 9c f4 aa 1e a8 cd a6 82 c7 59 0f 5e 9d e0 +0000320 bb fc 6c d6 99 23 eb 36 ad c6 c5 e1 d8 e1 e2 3e +0000330 d9 90 5a f7 91 5d 6f bc 33 6d 98 47 d2 7c 2e 2f +0000340 99 a4 25 72 85 49 2c be 0b 5b af 8f e5 6e 81 a6 +0000350 a3 5a 6f 39 53 3a ab 7a 8b 1e 26 f7 46 6c 7d 26 +0000360 53 b3 22 31 94 d3 83 f2 18 4d f5 92 33 27 53 97 +0000370 0f d3 e6 55 9c a6 c5 31 87 6f d3 f3 ae 39 6f 56 +0000380 10 7b ab 7e d0 b4 ca f2 b8 05 be 3f 0e 6e 5a 75 +0000390 ab 0c f5 37 0e ba 8e 75 71 7a aa ed 7a dd 6a 63 +00003a0 be 9b a0 97 27 6a 6f e7 d3 8b c4 7c ec d3 91 56 +00003b0 d9 ac 5e bf 16 42 2f 00 1f 93 a2 23 87 bd e2 59 +00003c0 a0 de 1a 66 c8 62 eb 55 8f 91 17 b4 61 42 7a 50 +00003d0 40 03 34 40 03 34 40 03 34 40 03 34 40 03 34 40 +00003e0 03 34 40 03 34 40 03 34 40 03 34 40 03 34 40 03 +00003f0 34 40 03 34 40 03 34 ff 85 86 90 8b ea 67 90 0d +0000400 e1 42 1b d2 61 d6 79 ec fd 3e 44 28 a4 51 6c 5c +0000410 fc d2 72 ca ba 82 18 46 16 61 cd 93 a9 0f d1 24 +0000420 17 99 e2 2c 71 16 84 0c c8 7a 13 0f 9a 5e c5 f0 +0000430 79 64 e2 12 4d c8 82 a1 81 19 2d aa 44 6d 87 54 +0000440 84 71 c1 f6 d4 ca 25 8c 77 b9 08 c7 c8 5e 10 8a +0000450 8f 61 ed 8c ba 30 1f 79 9a c7 60 34 2b b9 8c f8 +0000460 18 a6 83 1b e3 9f ad 79 fe fd 1b 8b f1 fc 41 6f +0000470 d4 13 1f e3 b8 83 ba 64 92 e7 eb e4 77 05 8f ba +0000480 fa 3b 00 00 ff ff 50 4b 07 08 a6 18 b1 91 5e 04 +0000490 00 00 e4 47 00 00 50 4b 01 02 14 00 14 00 08 00 +00004a0 08 00 00 00 00 00 a6 18 b1 91 5e 04 00 00 e4 47 +00004b0 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 +00004c0 00 00 00 00 62 69 67 67 65 72 2e 7a 69 70 50 4b +00004d0 05 06 00 00 00 00 01 00 01 00 38 00 00 00 96 04 +00004e0 00 00 00 00` + s = regexp.MustCompile(`[0-9a-f]{7}`).ReplaceAllString(s, "") + s = regexp.MustCompile(`\s+`).ReplaceAllString(s, "") + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +func returnBigZipBytes() (r io.ReaderAt, size int64) { + b := biggestZipBytes() + for i := 0; i < 2; i++ { + r, err := NewReader(bytes.NewReader(b), int64(len(b))) + if err != nil { + panic(err) + } + f, err := r.File[0].Open() + if err != nil { + panic(err) + } + b, err = ioutil.ReadAll(f) + if err != nil { + panic(err) + } + } + return bytes.NewReader(b), int64(len(b)) +} + func TestIssue8186(t *testing.T) { // Directory headers & data found in the TOC of a JAR file. dirEnts := []string{ diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go index 5db4bc6..c81bd40 100644 --- a/libgo/go/cmd/go/alldocs.go +++ b/libgo/go/cmd/go/alldocs.go @@ -93,7 +93,8 @@ and test commands: Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. -msan enable interoperation with memory sanitizer. - Supported only on linux/amd64. + Supported only on linux/amd64, + and only with Clang/LLVM as the host C compiler. -v print the names of packages as they are compiled. -work diff --git a/libgo/go/cmd/go/build.go b/libgo/go/cmd/go/build.go index 1b9a3e7..4382cf7 100644 --- a/libgo/go/cmd/go/build.go +++ b/libgo/go/cmd/go/build.go @@ -72,7 +72,8 @@ and test commands: Supported only on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64. -msan enable interoperation with memory sanitizer. - Supported only on linux/amd64. + Supported only on linux/amd64, + and only with Clang/LLVM as the host C compiler. -v print the names of packages as they are compiled. -work @@ -674,6 +675,7 @@ var ( goarch string goos string exeSuffix string + gopath []string ) func init() { @@ -682,6 +684,7 @@ func init() { if goos == "windows" { exeSuffix = ".exe" } + gopath = filepath.SplitList(buildContext.GOPATH) } // A builder holds global state about a build. @@ -1694,6 +1697,22 @@ func (b *builder) includeArgs(flag string, all []*action) []string { inc = append(inc, flag, b.work) // Finally, look in the installed package directories for each action. + // First add the package dirs corresponding to GOPATH entries + // in the original GOPATH order. + need := map[string]*build.Package{} + for _, a1 := range all { + if a1.p != nil && a1.pkgdir == a1.p.build.PkgRoot { + need[a1.p.build.Root] = a1.p.build + } + } + for _, root := range gopath { + if p := need[root]; p != nil && !incMap[p.PkgRoot] { + incMap[p.PkgRoot] = true + inc = append(inc, flag, p.PkgTargetRoot) + } + } + + // Then add anything that's left. for _, a1 := range all { if a1.p == nil { continue diff --git a/libgo/go/cmd/go/go_test.go b/libgo/go/cmd/go/go_test.go index a901ca8..39e0f3e 100644 --- a/libgo/go/cmd/go/go_test.go +++ b/libgo/go/cmd/go/go_test.go @@ -10,6 +10,7 @@ import ( "fmt" "go/build" "go/format" + "internal/race" "internal/testenv" "io" "io/ioutil" @@ -69,7 +70,11 @@ func TestMain(m *testing.M) { flag.Parse() if canRun { - out, err := exec.Command("go", "build", "-tags", "testgo", "-o", "testgo"+exeSuffix).CombinedOutput() + args := []string{"build", "-tags", "testgo", "-o", "testgo" + exeSuffix} + if race.Enabled { + args = append(args, "-race") + } + out, err := exec.Command("go", args...).CombinedOutput() if err != nil { fmt.Fprintf(os.Stderr, "building testgo failed: %v\n%s", err, out) os.Exit(2) @@ -657,6 +662,9 @@ func TestGoBuildDashAInDevBranch(t *testing.T) { tg.setenv("TESTGO_IS_GO_RELEASE", "0") tg.run("build", "-v", "-a", "math") tg.grepStderr("runtime", "testgo build -a math in dev branch DID NOT build runtime, but should have") + + // Everything is out of date. Rebuild to leave things in a better state. + tg.run("install", "std") } func TestGoBuildDashAInReleaseBranch(t *testing.T) { @@ -672,11 +680,80 @@ func TestGoBuildDashAInReleaseBranch(t *testing.T) { tg.grepStderr("runtime", "testgo build -a math in release branch DID NOT build runtime, but should have") // Now runtime.a is updated (newer mtime), so everything would look stale if not for being a release. - // tg.run("build", "-v", "net/http") tg.grepStderrNot("strconv", "testgo build -v net/http in release branch with newer runtime.a DID build strconv but should not have") tg.grepStderrNot("golang.org/x/net/http2/hpack", "testgo build -v net/http in release branch with newer runtime.a DID build .../golang.org/x/net/http2/hpack but should not have") tg.grepStderrNot("net/http", "testgo build -v net/http in release branch with newer runtime.a DID build net/http but should not have") + + // Everything is out of date. Rebuild to leave things in a better state. + tg.run("install", "std") +} + +func TestNewReleaseRebuildsStalePackagesInGOPATH(t *testing.T) { + if testing.Short() { + t.Skip("don't rebuild the standard library in short mode") + } + + tg := testgo(t) + defer tg.cleanup() + + addNL := func(name string) (restore func()) { + data, err := ioutil.ReadFile(name) + if err != nil { + t.Fatal(err) + } + old := data + data = append(data, '\n') + if err := ioutil.WriteFile(name, append(data, '\n'), 0666); err != nil { + t.Fatal(err) + } + tg.sleep() + return func() { + if err := ioutil.WriteFile(name, old, 0666); err != nil { + t.Fatal(err) + } + } + } + + tg.setenv("TESTGO_IS_GO_RELEASE", "1") + + tg.tempFile("d1/src/p1/p1.go", `package p1`) + tg.setenv("GOPATH", tg.path("d1")) + tg.run("install", "-a", "p1") + tg.wantNotStale("p1", "./testgo list claims p1 is stale, incorrectly") + tg.sleep() + + // Changing mtime and content of runtime/internal/sys/sys.go + // should have no effect: we're in a release, which doesn't rebuild + // for general mtime or content changes. + sys := runtime.GOROOT() + "/src/runtime/internal/sys/sys.go" + restore := addNL(sys) + defer restore() + tg.wantNotStale("p1", "./testgo list claims p1 is stale, incorrectly, after updating runtime/internal/sys/sys.go") + restore() + tg.wantNotStale("p1", "./testgo list claims p1 is stale, incorrectly, after restoring runtime/internal/sys/sys.go") + + // But changing runtime/internal/sys/zversion.go should have an effect: + // that's how we tell when we flip from one release to another. + zversion := runtime.GOROOT() + "/src/runtime/internal/sys/zversion.go" + restore = addNL(zversion) + defer restore() + tg.wantStale("p1", "./testgo list claims p1 is NOT stale, incorrectly, after changing to new release") + restore() + tg.wantNotStale("p1", "./testgo list claims p1 is stale, incorrectly, after changing back to old release") + addNL(zversion) + tg.wantStale("p1", "./testgo list claims p1 is NOT stale, incorrectly, after changing again to new release") + tg.run("install", "p1") + tg.wantNotStale("p1", "./testgo list claims p1 is stale after building with new release") + + // Restore to "old" release. + restore() + tg.wantStale("p1", "./testgo list claims p1 is NOT stale, incorrectly, after changing to old release after new build") + tg.run("install", "p1") + tg.wantNotStale("p1", "./testgo list claims p1 is stale after building with old release") + + // Everything is out of date. Rebuild to leave things in a better state. + tg.run("install", "std") } func TestGoListStandard(t *testing.T) { @@ -756,8 +833,8 @@ func TestGoInstallRebuildsStalePackagesInOtherGOPATH(t *testing.T) { sep := string(filepath.ListSeparator) tg.setenv("GOPATH", tg.path("d1")+sep+tg.path("d2")) tg.run("install", "p1") - tg.wantNotStale("p1", "./testgo list mypkg claims p1 is stale, incorrectly") - tg.wantNotStale("p2", "./testgo list mypkg claims p2 is stale, incorrectly") + tg.wantNotStale("p1", "./testgo list claims p1 is stale, incorrectly") + tg.wantNotStale("p2", "./testgo list claims p2 is stale, incorrectly") tg.sleep() if f, err := os.OpenFile(tg.path("d2/src/p2/p2.go"), os.O_WRONLY|os.O_APPEND, 0); err != nil { t.Fatal(err) @@ -766,12 +843,12 @@ func TestGoInstallRebuildsStalePackagesInOtherGOPATH(t *testing.T) { } else { tg.must(f.Close()) } - tg.wantStale("p2", "./testgo list mypkg claims p2 is NOT stale, incorrectly") - tg.wantStale("p1", "./testgo list mypkg claims p1 is NOT stale, incorrectly") + tg.wantStale("p2", "./testgo list claims p2 is NOT stale, incorrectly") + tg.wantStale("p1", "./testgo list claims p1 is NOT stale, incorrectly") tg.run("install", "p1") - tg.wantNotStale("p2", "./testgo list mypkg claims p2 is stale after reinstall, incorrectly") - tg.wantNotStale("p1", "./testgo list mypkg claims p1 is stale after reinstall, incorrectly") + tg.wantNotStale("p2", "./testgo list claims p2 is stale after reinstall, incorrectly") + tg.wantNotStale("p1", "./testgo list claims p1 is stale after reinstall, incorrectly") } func TestGoInstallDetectsRemovedFiles(t *testing.T) { @@ -1621,7 +1698,7 @@ func TestGoTestDashOWritesBinary(t *testing.T) { } // Issue 4568. -func TestSymlinksDoNotConfuseGoList(t *testing.T) { +func TestSymlinksList(t *testing.T) { switch runtime.GOOS { case "plan9", "windows": t.Skipf("skipping symlink test on %s", runtime.GOOS) @@ -1640,6 +1717,58 @@ func TestSymlinksDoNotConfuseGoList(t *testing.T) { } } +// Issue 14054. +func TestSymlinksVendor(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("skipping symlink test on %s", runtime.GOOS) + } + + tg := testgo(t) + defer tg.cleanup() + tg.setenv("GO15VENDOREXPERIMENT", "1") + tg.tempDir("gopath/src/dir1/vendor/v") + tg.tempFile("gopath/src/dir1/p.go", "package main\nimport _ `v`\nfunc main(){}") + tg.tempFile("gopath/src/dir1/vendor/v/v.go", "package v") + tg.must(os.Symlink(tg.path("gopath/src/dir1"), tg.path("symdir1"))) + tg.setenv("GOPATH", tg.path("gopath")) + tg.cd(tg.path("symdir1")) + tg.run("list", "-f", "{{.Root}}", ".") + if strings.TrimSpace(tg.getStdout()) != tg.path("gopath") { + t.Error("list confused by symlinks") + } + + // All of these should succeed, not die in vendor-handling code. + tg.run("run", "p.go") + tg.run("build") + tg.run("install") +} + +func TestSymlinksInternal(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("skipping symlink test on %s", runtime.GOOS) + } + + tg := testgo(t) + defer tg.cleanup() + tg.tempDir("gopath/src/dir1/internal/v") + tg.tempFile("gopath/src/dir1/p.go", "package main\nimport _ `dir1/internal/v`\nfunc main(){}") + tg.tempFile("gopath/src/dir1/internal/v/v.go", "package v") + tg.must(os.Symlink(tg.path("gopath/src/dir1"), tg.path("symdir1"))) + tg.setenv("GOPATH", tg.path("gopath")) + tg.cd(tg.path("symdir1")) + tg.run("list", "-f", "{{.Root}}", ".") + if strings.TrimSpace(tg.getStdout()) != tg.path("gopath") { + t.Error("list confused by symlinks") + } + + // All of these should succeed, not die in internal-handling code. + tg.run("run", "p.go") + tg.run("build") + tg.run("install") +} + // Issue 4515. func TestInstallWithTags(t *testing.T) { tg := testgo(t) @@ -2441,6 +2570,59 @@ func TestGoInstallShadowedGOPATH(t *testing.T) { tg.grepStderr("no install location for.*gopath2.src.test: hidden by .*gopath1.src.test", "missing error") } +func TestGoBuildGOPATHOrder(t *testing.T) { + // golang.org/issue/14176#issuecomment-179895769 + // golang.org/issue/14192 + // -I arguments to compiler could end up not in GOPATH order, + // leading to unexpected import resolution in the compiler. + // This is still not a complete fix (see golang.org/issue/14271 and next test) + // but it is clearly OK and enough to fix both of the two reported + // instances of the underlying problem. It will have to do for now. + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + tg.setenv("GOPATH", tg.path("p1")+string(filepath.ListSeparator)+tg.path("p2")) + + tg.tempFile("p1/src/foo/foo.go", "package foo\n") + tg.tempFile("p2/src/baz/baz.go", "package baz\n") + tg.tempFile("p2/pkg/"+runtime.GOOS+"_"+runtime.GOARCH+"/foo.a", "bad\n") + tg.tempFile("p1/src/bar/bar.go", ` + package bar + import _ "baz" + import _ "foo" + `) + + tg.run("install", "-x", "bar") +} + +func TestGoBuildGOPATHOrderBroken(t *testing.T) { + // This test is known not to work. + // See golang.org/issue/14271. + t.Skip("golang.org/issue/14271") + + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + + tg.tempFile("p1/src/foo/foo.go", "package foo\n") + tg.tempFile("p2/src/baz/baz.go", "package baz\n") + tg.tempFile("p1/pkg/"+runtime.GOOS+"_"+runtime.GOARCH+"/baz.a", "bad\n") + tg.tempFile("p2/pkg/"+runtime.GOOS+"_"+runtime.GOARCH+"/foo.a", "bad\n") + tg.tempFile("p1/src/bar/bar.go", ` + package bar + import _ "baz" + import _ "foo" + `) + + colon := string(filepath.ListSeparator) + tg.setenv("GOPATH", tg.path("p1")+colon+tg.path("p2")) + tg.run("install", "-x", "bar") + + tg.setenv("GOPATH", tg.path("p2")+colon+tg.path("p1")) + tg.run("install", "-x", "bar") +} + func TestIssue11709(t *testing.T) { tg := testgo(t) defer tg.cleanup() @@ -2558,3 +2740,22 @@ func TestIssue13655(t *testing.T) { tg.grepStdout("runtime/internal/sys", "did not find required dependency of "+pkg+" on runtime/internal/sys") } } + +// For issue 14337. +func TestParallelTest(t *testing.T) { + tg := testgo(t) + defer tg.cleanup() + tg.makeTempdir() + const testSrc = `package package_test + import ( + "testing" + ) + func TestTest(t *testing.T) { + }` + tg.tempFile("src/p1/p1_test.go", strings.Replace(testSrc, "package_test", "p1_test", 1)) + tg.tempFile("src/p2/p2_test.go", strings.Replace(testSrc, "package_test", "p2_test", 1)) + tg.tempFile("src/p3/p3_test.go", strings.Replace(testSrc, "package_test", "p3_test", 1)) + tg.tempFile("src/p4/p4_test.go", strings.Replace(testSrc, "package_test", "p4_test", 1)) + tg.setenv("GOPATH", tg.path(".")) + tg.run("test", "-p=4", "p1", "p2", "p3", "p4") +} diff --git a/libgo/go/cmd/go/main.go b/libgo/go/cmd/go/main.go index c8697ff..f9b979d 100644 --- a/libgo/go/cmd/go/main.go +++ b/libgo/go/cmd/go/main.go @@ -454,7 +454,9 @@ func envForDir(dir string, base []string) []string { // mergeEnvLists merges the two environment lists such that // variables with the same name in "in" replace those in "out". +// This always returns a newly allocated slice. func mergeEnvLists(in, out []string) []string { + out = append([]string(nil), out...) NextVar: for _, inkv := range in { k := strings.SplitAfterN(inkv, "=", 2)[0] @@ -524,6 +526,15 @@ func hasFilePathPrefix(s, prefix string) bool { } } +// expandPath returns the symlink-expanded form of path. +func expandPath(p string) string { + x, err := filepath.EvalSymlinks(p) + if err == nil { + return x + } + return p +} + // treeCanMatchPattern(pattern)(name) reports whether // name or children of name can possibly match pattern. // Pattern is the same limited glob accepted by matchPattern. diff --git a/libgo/go/cmd/go/pkg.go b/libgo/go/cmd/go/pkg.go index 73ea206..373dade 100644 --- a/libgo/go/cmd/go/pkg.go +++ b/libgo/go/cmd/go/pkg.go @@ -419,11 +419,18 @@ func vendoredImportPath(parent *Package, path string) (found string) { if parent == nil || parent.Root == "" || !go15VendorExperiment { return path } + dir := filepath.Clean(parent.Dir) root := filepath.Join(parent.Root, "src") + if !hasFilePathPrefix(dir, root) { + // Look for symlinks before reporting error. + dir = expandPath(dir) + root = expandPath(root) + } if !hasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator { fatalf("invalid vendoredImportPath: dir=%q root=%q separator=%q", dir, root, string(filepath.Separator)) } + vpath := "vendor/" + path for i := len(dir); i >= len(root); i-- { if i < len(dir) && dir[i] != filepath.Separator { @@ -537,6 +544,13 @@ func disallowInternal(srcDir string, p *Package, stk *importStack) *Package { return p } + // Look for symlinks before reporting error. + srcDir = expandPath(srcDir) + parent = expandPath(parent) + if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { + return p + } + // Internal is present, and srcDir is outside parent's tree. Not allowed. perr := *p perr.Error = &PackageError{ @@ -634,6 +648,13 @@ func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Pack return p } + // Look for symlinks before reporting error. + srcDir = expandPath(srcDir) + parent = expandPath(parent) + if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { + return p + } + // Vendor is present, and srcDir is outside parent's tree. Not allowed. perr := *p perr.Error = &PackageError{ @@ -957,7 +978,7 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } } } - if p.Standard && !p1.Standard && p.Error == nil { + if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil { p.Error = &PackageError{ ImportStack: stk.copy(), Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath), @@ -1532,11 +1553,14 @@ func computeBuildID(p *Package) { fmt.Fprintf(h, "file %s\n", file) } - // Include the content of runtime/zversion.go in the hash + // Include the content of runtime/internal/sys/zversion.go in the hash // for package runtime. This will give package runtime a // different build ID in each Go release. - if p.Standard && p.ImportPath == "runtime" { - data, _ := ioutil.ReadFile(filepath.Join(p.Dir, "zversion.go")) + if p.Standard && p.ImportPath == "runtime/internal/sys" { + data, err := ioutil.ReadFile(filepath.Join(p.Dir, "zversion.go")) + if err != nil { + fatalf("go: %s", err) + } fmt.Fprintf(h, "zversion %q\n", string(data)) } diff --git a/libgo/go/cmd/go/vcs.go b/libgo/go/cmd/go/vcs.go index 074dd8b..342edee 100644 --- a/libgo/go/cmd/go/vcs.go +++ b/libgo/go/cmd/go/vcs.go @@ -122,7 +122,7 @@ var vcsGit = &vcsCmd{ name: "Git", cmd: "git", - createCmd: []string{"clone {repo} {dir}", "-C {dir} submodule update --init --recursive"}, + createCmd: []string{"clone {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, tagCmd: []tagCmd{ @@ -335,6 +335,15 @@ func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) args[i] = expand(m, arg) } + if len(args) >= 2 && args[0] == "-go-internal-cd" { + if filepath.IsAbs(args[1]) { + dir = args[1] + } else { + dir = filepath.Join(dir, args[1]) + } + args = args[2:] + } + _, err := exec.LookPath(v.cmd) if err != nil { fmt.Fprintf(os.Stderr, diff --git a/libgo/go/go/constant/value.go b/libgo/go/go/constant/value.go index 6305810..310814d 100644 --- a/libgo/go/go/constant/value.go +++ b/libgo/go/go/constant/value.go @@ -96,7 +96,7 @@ func (x stringVal) String() string { // only the first maxLen-3 runes; then add "...". i := 0 for n := 0; n < maxLen-3; n++ { - _, size := utf8.DecodeRuneInString(s) + _, size := utf8.DecodeRuneInString(s[i:]) i += size } s = s[:i] + "..." diff --git a/libgo/go/go/constant/value_test.go b/libgo/go/go/constant/value_test.go index de1ab02..dbd96c0 100644 --- a/libgo/go/go/constant/value_test.go +++ b/libgo/go/go/constant/value_test.go @@ -204,6 +204,7 @@ func eql(x, y Value) bool { // String tests var xxx = strings.Repeat("x", 68) +var issue14262 = `"بموجب الشروط التالية نسب المصنف — يجب عليك أن تنسب العمل بالطريقة التي تحددها المؤلف أو المرخص (ولكن ليس بأي حال من الأحوال أن توحي وتقترح بتحول أو استخدامك للعمل). المشاركة على قدم المساواة — إذا كنت يعدل ، والتغيير ، أو الاستفادة من هذا العمل ، قد ينتج عن توزيع العمل إلا في ظل تشابه او تطابق فى واحد لهذا الترخيص."` var stringTests = []struct { input, short, exact string @@ -225,6 +226,7 @@ var stringTests = []struct { {`"` + xxx + `xx"`, `"` + xxx + `xx"`, `"` + xxx + `xx"`}, {`"` + xxx + `xxx"`, `"` + xxx + `...`, `"` + xxx + `xxx"`}, {`"` + xxx + xxx + `xxx"`, `"` + xxx + `...`, `"` + xxx + xxx + `xxx"`}, + {issue14262, `"بموجب الشروط التالية نسب المصنف — يجب عليك أن تنسب العمل بالطريقة ال...`, issue14262}, // Int {"0", "0", "0"}, diff --git a/libgo/go/go/internal/gcimporter/gcimporter.go b/libgo/go/go/internal/gcimporter/gcimporter.go index 0ef8eb4..d70ec08 100644 --- a/libgo/go/go/internal/gcimporter/gcimporter.go +++ b/libgo/go/go/internal/gcimporter/gcimporter.go @@ -31,7 +31,8 @@ var pkgExts = [...]string{".a", ".o"} // FindPkg returns the filename and unique package id for an import // path based on package information provided by build.Import (using -// the build.Default build.Context). +// the build.Default build.Context). A relative srcDir is interpreted +// relative to the current working directory. // If no file was found, an empty filename is returned. // func FindPkg(path, srcDir string) (filename, id string) { @@ -44,6 +45,9 @@ func FindPkg(path, srcDir string) (filename, id string) { default: // "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x" // Don't require the source files to be present. + if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282 + srcDir = abs + } bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary) if bp.PkgObj == "" { return diff --git a/libgo/go/go/types/expr.go b/libgo/go/go/types/expr.go index 942d3fd..f7c4a17 100644 --- a/libgo/go/go/types/expr.go +++ b/libgo/go/go/types/expr.go @@ -184,7 +184,8 @@ func roundFloat64(x constant.Value) constant.Value { // provided (only needed for int/uint sizes). // // If rounded != nil, *rounded is set to the rounded value of x for -// representable floating-point values; it is left alone otherwise. +// representable floating-point and complex values, and to an Int +// value for integer values; it is left alone otherwise. // It is ok to provide the addressof the first argument for rounded. func representableConst(x constant.Value, conf *Config, typ *Basic, rounded *constant.Value) bool { if x.Kind() == constant.Unknown { @@ -197,6 +198,9 @@ func representableConst(x constant.Value, conf *Config, typ *Basic, rounded *con if x.Kind() != constant.Int { return false } + if rounded != nil { + *rounded = x + } if x, ok := constant.Int64Val(x); ok { switch typ.kind { case Int: @@ -808,8 +812,6 @@ func (check *Checker) binary(x *operand, e *ast.BinaryExpr, lhs, rhs ast.Expr, o typ := x.typ.Underlying().(*Basic) // force integer division of integer operands if op == token.QUO && isInteger(typ) { - xval = constant.ToInt(xval) - yval = constant.ToInt(yval) op = token.QUO_ASSIGN } x.val = constant.BinaryOp(xval, op, yval) diff --git a/libgo/go/go/types/resolver.go b/libgo/go/go/types/resolver.go index 14148a5..1536df5 100644 --- a/libgo/go/go/types/resolver.go +++ b/libgo/go/go/types/resolver.go @@ -483,11 +483,9 @@ func pkgName(path string) string { // (Per the go/build package dependency tests, we cannot import // path/filepath and simply use filepath.Dir.) func dir(path string) string { - if i := strings.LastIndexAny(path, "/\\"); i >= 0 { - path = path[:i] + if i := strings.LastIndexAny(path, `/\`); i > 0 { + return path[:i] } - if path == "" { - path = "." - } - return path + // i <= 0 + return "." } diff --git a/libgo/go/net/http/clientserver_test.go b/libgo/go/net/http/clientserver_test.go index 3c87fd0..aa2473a 100644 --- a/libgo/go/net/http/clientserver_test.go +++ b/libgo/go/net/http/clientserver_test.go @@ -1001,9 +1001,11 @@ func TestTransportDiscardsUnneededConns(t *testing.T) { } // tests that Transport doesn't retain a pointer to the provided request. -func TestTransportGCRequest_h1(t *testing.T) { testTransportGCRequest(t, h1Mode) } -func TestTransportGCRequest_h2(t *testing.T) { testTransportGCRequest(t, h2Mode) } -func testTransportGCRequest(t *testing.T, h2 bool) { +func TestTransportGCRequest_Body_h1(t *testing.T) { testTransportGCRequest(t, h1Mode, true) } +func TestTransportGCRequest_Body_h2(t *testing.T) { testTransportGCRequest(t, h2Mode, true) } +func TestTransportGCRequest_NoBody_h1(t *testing.T) { testTransportGCRequest(t, h1Mode, false) } +func TestTransportGCRequest_NoBody_h2(t *testing.T) { testTransportGCRequest(t, h2Mode, false) } +func testTransportGCRequest(t *testing.T, h2, body bool) { if runtime.Compiler == "gccgo" { t.Skip("skipping on gccgo because conservative GC means that finalizer may never run") } @@ -1011,7 +1013,9 @@ func testTransportGCRequest(t *testing.T, h2 bool) { defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { ioutil.ReadAll(r.Body) - io.WriteString(w, "Hello.") + if body { + io.WriteString(w, "Hello.") + } })) defer cst.close() diff --git a/libgo/go/net/http/h2_bundle.go b/libgo/go/net/http/h2_bundle.go index e723629..4e19b3e 100644 --- a/libgo/go/net/http/h2_bundle.go +++ b/libgo/go/net/http/h2_bundle.go @@ -1,5 +1,5 @@ -// Code generated by golang.org/x/tools/cmd/bundle command: -// $ bundle golang.org/x/net/http2 net/http http2 +// Code generated by golang.org/x/tools/cmd/bundle. +//go:generate bundle -o h2_bundle.go -prefix http2 -import golang.org/x/net/http2/hpack=internal/golang.org/x/net/http2/hpack golang.org/x/net/http2 // Package http2 implements the HTTP/2 protocol. // @@ -2331,6 +2331,10 @@ var http2isTokenTable = [127]bool{ '~': true, } +type http2connectionStater interface { + ConnectionState() tls.ConnectionState +} + // pipe is a goroutine-safe io.Reader/io.Writer pair. It's like // io.Pipe except there are no PipeReader/PipeWriter halves, and the // underlying buffer is an interface. (io.Pipe is always unbuffered) @@ -2593,28 +2597,76 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { if http2testHookOnConn != nil { http2testHookOnConn() } - conf.handleConn(hs, c, h) + conf.ServeConn(c, &http2ServeConnOpts{ + Handler: h, + BaseConfig: hs, + }) } s.TLSNextProto[http2NextProtoTLS] = protoHandler s.TLSNextProto["h2-14"] = protoHandler return nil } -func (srv *http2Server) handleConn(hs *Server, c net.Conn, h Handler) { +// ServeConnOpts are options for the Server.ServeConn method. +type http2ServeConnOpts struct { + // BaseConfig optionally sets the base configuration + // for values. If nil, defaults are used. + BaseConfig *Server + + // Handler specifies which handler to use for processing + // requests. If nil, BaseConfig.Handler is used. If BaseConfig + // or BaseConfig.Handler is nil, http.DefaultServeMux is used. + Handler Handler +} + +func (o *http2ServeConnOpts) baseConfig() *Server { + if o != nil && o.BaseConfig != nil { + return o.BaseConfig + } + return new(Server) +} + +func (o *http2ServeConnOpts) handler() Handler { + if o != nil { + if o.Handler != nil { + return o.Handler + } + if o.BaseConfig != nil && o.BaseConfig.Handler != nil { + return o.BaseConfig.Handler + } + } + return DefaultServeMux +} + +// ServeConn serves HTTP/2 requests on the provided connection and +// blocks until the connection is no longer readable. +// +// ServeConn starts speaking HTTP/2 assuming that c has not had any +// reads or writes. It writes its initial settings frame and expects +// to be able to read the preface and settings frame from the +// client. If c has a ConnectionState method like a *tls.Conn, the +// ConnectionState is used to verify the TLS ciphersuite and to set +// the Request.TLS field in Handlers. +// +// ServeConn does not support h2c by itself. Any h2c support must be +// implemented in terms of providing a suitably-behaving net.Conn. +// +// The opts parameter is optional. If nil, default values are used. +func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { sc := &http2serverConn{ - srv: srv, - hs: hs, + srv: s, + hs: opts.baseConfig(), conn: c, remoteAddrStr: c.RemoteAddr().String(), bw: http2newBufferedWriter(c), - handler: h, + handler: opts.handler(), streams: make(map[uint32]*http2stream), readFrameCh: make(chan http2readFrameResult), wantWriteFrameCh: make(chan http2frameWriteMsg, 8), wroteFrameCh: make(chan http2frameWriteResult, 1), bodyReadCh: make(chan http2bodyReadMsg), doneServing: make(chan struct{}), - advMaxStreams: srv.maxConcurrentStreams(), + advMaxStreams: s.maxConcurrentStreams(), writeSched: http2writeScheduler{ maxFrameSize: http2initialMaxFrameSize, }, @@ -2630,10 +2682,10 @@ func (srv *http2Server) handleConn(hs *Server, c net.Conn, h Handler) { sc.hpackDecoder.SetMaxStringLength(sc.maxHeaderStringLen()) fr := http2NewFramer(sc.bw, c) - fr.SetMaxReadFrameSize(srv.maxReadFrameSize()) + fr.SetMaxReadFrameSize(s.maxReadFrameSize()) sc.framer = fr - if tc, ok := c.(*tls.Conn); ok { + if tc, ok := c.(http2connectionStater); ok { sc.tlsState = new(tls.ConnectionState) *sc.tlsState = tc.ConnectionState() @@ -2646,7 +2698,7 @@ func (srv *http2Server) handleConn(hs *Server, c net.Conn, h Handler) { } - if !srv.PermitProhibitedCipherSuites && http2isBadCipher(sc.tlsState.CipherSuite) { + if !s.PermitProhibitedCipherSuites && http2isBadCipher(sc.tlsState.CipherSuite) { sc.rejectConn(http2ErrCodeInadequateSecurity, fmt.Sprintf("Prohibited TLS 1.2 Cipher Suite: %x", sc.tlsState.CipherSuite)) return @@ -2851,8 +2903,6 @@ func (sc *http2serverConn) logf(format string, args ...interface{}) { } } -var http2uintptrType = reflect.TypeOf(uintptr(0)) - // errno returns v's underlying uintptr, else 0. // // TODO: remove this helper function once http2 can use build @@ -4220,7 +4270,9 @@ func (rws *http2responseWriterState) declareTrailer(k string) { return } - rws.trailers = append(rws.trailers, k) + if !http2strSliceContains(rws.trailers, k) { + rws.trailers = append(rws.trailers, k) + } } // writeChunk writes chunks from the bufio.Writer. But because @@ -4288,6 +4340,10 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { return 0, nil } + if rws.handlerDone { + rws.promoteUndeclaredTrailers() + } + endStream := rws.handlerDone && !rws.hasTrailers() if len(p) > 0 || endStream { @@ -4308,6 +4364,53 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { return len(p), nil } +// TrailerPrefix is a magic prefix for ResponseWriter.Header map keys +// that, if present, signals that the map entry is actually for +// the response trailers, and not the response headers. The prefix +// is stripped after the ServeHTTP call finishes and the values are +// sent in the trailers. +// +// This mechanism is intended only for trailers that are not known +// prior to the headers being written. If the set of trailers is fixed +// or known before the header is written, the normal Go trailers mechanism +// is preferred: +// https://golang.org/pkg/net/http/#ResponseWriter +// https://golang.org/pkg/net/http/#example_ResponseWriter_trailers +const http2TrailerPrefix = "Trailer:" + +// promoteUndeclaredTrailers permits http.Handlers to set trailers +// after the header has already been flushed. Because the Go +// ResponseWriter interface has no way to set Trailers (only the +// Header), and because we didn't want to expand the ResponseWriter +// interface, and because nobody used trailers, and because RFC 2616 +// says you SHOULD (but not must) predeclare any trailers in the +// header, the official ResponseWriter rules said trailers in Go must +// be predeclared, and then we reuse the same ResponseWriter.Header() +// map to mean both Headers and Trailers. When it's time to write the +// Trailers, we pick out the fields of Headers that were declared as +// trailers. That worked for a while, until we found the first major +// user of Trailers in the wild: gRPC (using them only over http2), +// and gRPC libraries permit setting trailers mid-stream without +// predeclarnig them. So: change of plans. We still permit the old +// way, but we also permit this hack: if a Header() key begins with +// "Trailer:", the suffix of that key is a Trailer. Because ':' is an +// invalid token byte anyway, there is no ambiguity. (And it's already +// filtered out) It's mildly hacky, but not terrible. +// +// This method runs after the Handler is done and promotes any Header +// fields to be trailers. +func (rws *http2responseWriterState) promoteUndeclaredTrailers() { + for k, vv := range rws.handlerHeader { + if !strings.HasPrefix(k, http2TrailerPrefix) { + continue + } + trailerKey := strings.TrimPrefix(k, http2TrailerPrefix) + rws.declareTrailer(trailerKey) + rws.handlerHeader[CanonicalHeaderKey(trailerKey)] = vv + } + sort.Strings(rws.trailers) +} + func (w *http2responseWriter) Flush() { rws := w.rws if rws == nil { @@ -4823,10 +4926,7 @@ func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { cc.henc = hpack.NewEncoder(&cc.hbuf) - type connectionStater interface { - ConnectionState() tls.ConnectionState - } - if cs, ok := c.(connectionStater); ok { + if cs, ok := c.(http2connectionStater); ok { state := cs.ConnectionState() cc.tlsState = &state } @@ -4977,7 +5077,27 @@ func (cc *http2ClientConn) responseHeaderTimeout() time.Duration { return 0 } +// checkConnHeaders checks whether req has any invalid connection-level headers. +// per RFC 7540 section 8.1.2.2: Connection-Specific Header Fields. +// Certain headers are special-cased as okay but not transmitted later. +func http2checkConnHeaders(req *Request) error { + if v := req.Header.Get("Upgrade"); v != "" { + return errors.New("http2: invalid Upgrade request header") + } + if v := req.Header.Get("Transfer-Encoding"); (v != "" && v != "chunked") || len(req.Header["Transfer-Encoding"]) > 1 { + return errors.New("http2: invalid Transfer-Encoding request header") + } + if v := req.Header.Get("Connection"); (v != "" && v != "close" && v != "keep-alive") || len(req.Header["Connection"]) > 1 { + return errors.New("http2: invalid Connection request header") + } + return nil +} + func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { + if err := http2checkConnHeaders(req); err != nil { + return nil, err + } + trailers, err := http2commaSeparatedTrailers(req) if err != nil { return nil, err @@ -5283,10 +5403,14 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail var didUA bool for k, vv := range req.Header { lowKey := strings.ToLower(k) - if lowKey == "host" || lowKey == "content-length" { + switch lowKey { + case "host", "content-length": + continue - } - if lowKey == "user-agent" { + case "connection", "proxy-connection", "transfer-encoding", "upgrade": + + continue + case "user-agent": didUA = true if len(vv) < 1 { @@ -5394,8 +5518,9 @@ func (cc *http2ClientConn) streamByID(id uint32, andRemove bool) *http2clientStr // clientConnReadLoop is the state owned by the clientConn's frame-reading readLoop. type http2clientConnReadLoop struct { - cc *http2ClientConn - activeRes map[uint32]*http2clientStream // keyed by streamID + cc *http2ClientConn + activeRes map[uint32]*http2clientStream // keyed by streamID + closeWhenIdle bool hdec *hpack.Decoder @@ -5452,7 +5577,7 @@ func (rl *http2clientConnReadLoop) cleanup() { func (rl *http2clientConnReadLoop) run() error { cc := rl.cc - closeWhenIdle := cc.t.disableKeepAlives() + rl.closeWhenIdle = cc.t.disableKeepAlives() gotReply := false for { f, err := cc.fr.ReadFrame() @@ -5501,7 +5626,7 @@ func (rl *http2clientConnReadLoop) run() error { if err != nil { return err } - if closeWhenIdle && gotReply && maybeIdle && len(rl.activeRes) == 0 { + if rl.closeWhenIdle && gotReply && maybeIdle && len(rl.activeRes) == 0 { cc.closeIfIdle() } } @@ -5611,10 +5736,10 @@ func (rl *http2clientConnReadLoop) processHeaderBlockFragment(frag []byte, strea res.ContentLength = -1 res.Body = &http2gzipReader{body: res.Body} } + rl.activeRes[cs.ID] = cs } cs.resTrailer = &res.Trailer - rl.activeRes[cs.ID] = cs cs.resc <- http2resAndError{res: res} rl.nextRes = nil return nil @@ -5752,6 +5877,9 @@ func (rl *http2clientConnReadLoop) endStream(cs *http2clientStream) { } cs.bufPipe.closeWithErrorAndCode(err, code) delete(rl.activeRes, cs.ID) + if cs.req.Close || cs.req.Header.Get("Connection") == "close" { + rl.closeWhenIdle = true + } } func (cs *http2clientStream) copyTrailers() { @@ -6013,13 +6141,18 @@ func (rt http2erringRoundTripper) RoundTrip(*Request) (*Response, error) { retur // call gzip.NewReader on the first call to Read type http2gzipReader struct { body io.ReadCloser // underlying Response.Body - zr io.Reader // lazily-initialized gzip reader + zr *gzip.Reader // lazily-initialized gzip reader + zerr error // sticky error } func (gz *http2gzipReader) Read(p []byte) (n int, err error) { + if gz.zerr != nil { + return 0, gz.zerr + } if gz.zr == nil { gz.zr, err = gzip.NewReader(gz.body) if err != nil { + gz.zerr = err return 0, err } } @@ -6258,8 +6391,16 @@ func http2encodeHeaders(enc *hpack.Encoder, h Header, keys []string) { for _, k := range keys { vv := h[k] k = http2lowerHeader(k) + if !http2validHeaderFieldName(k) { + + continue + } isTE := k == "transfer-encoding" for _, v := range vv { + if !http2validHeaderFieldValue(v) { + + continue + } if isTE && v != "trailers" { continue diff --git a/libgo/go/net/http/httptest/server.go b/libgo/go/net/http/httptest/server.go index 5c19c0c..bbe3233 100644 --- a/libgo/go/net/http/httptest/server.go +++ b/libgo/go/net/http/httptest/server.go @@ -202,10 +202,31 @@ func (s *Server) logCloseHangDebugInfo() { // CloseClientConnections closes any open HTTP connections to the test Server. func (s *Server) CloseClientConnections() { + var conns int + ch := make(chan bool) + s.mu.Lock() - defer s.mu.Unlock() for c := range s.conns { - s.closeConn(c) + conns++ + s.closeConnChan(c, ch) + } + s.mu.Unlock() + + // Wait for outstanding closes to finish. + // + // Out of paranoia for making a late change in Go 1.6, we + // bound how long this can wait, since golang.org/issue/14291 + // isn't fully understood yet. At least this should only be used + // in tests. + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + for i := 0; i < conns; i++ { + select { + case <-ch: + case <-timer.C: + // Too slow. Give up. + return + } } } @@ -267,9 +288,13 @@ func (s *Server) wrap() { } } -// closeConn closes c. Except on plan9, which is special. See comment below. +// closeConn closes c. // s.mu must be held. -func (s *Server) closeConn(c net.Conn) { +func (s *Server) closeConn(c net.Conn) { s.closeConnChan(c, nil) } + +// closeConnChan is like closeConn, but takes an optional channel to receive a value +// when the goroutine closing c is done. +func (s *Server) closeConnChan(c net.Conn, done chan<- bool) { if runtime.GOOS == "plan9" { // Go's Plan 9 net package isn't great at unblocking reads when // their underlying TCP connections are closed. Don't trust @@ -278,7 +303,21 @@ func (s *Server) closeConn(c net.Conn) { // resources if the syscall doesn't end up returning. Oh well. s.forgetConn(c) } - go c.Close() + + // Somewhere in the chaos of https://golang.org/cl/15151 we found that + // some types of conns were blocking in Close too long (or deadlocking?) + // and we had to call Close in a goroutine. I (bradfitz) forget what + // that was at this point, but I suspect it was *tls.Conns, which + // were later fixed in https://golang.org/cl/18572, so this goroutine + // is _probably_ unnecessary now. But it's too late in Go 1.6 too remove + // it with confidence. + // TODO(bradfitz): try to remove it for Go 1.7. (golang.org/issue/14291) + go func() { + c.Close() + if done != nil { + done <- true + } + }() } // forgetConn removes c from the set of tracked conns and decrements it from the diff --git a/libgo/go/net/http/httptest/server_test.go b/libgo/go/net/http/httptest/server_test.go index 6ffc671..c9606f2 100644 --- a/libgo/go/net/http/httptest/server_test.go +++ b/libgo/go/net/http/httptest/server_test.go @@ -84,3 +84,17 @@ func TestServerCloseBlocking(t *testing.T) { ts.Close() // test we don't hang here forever. } + +// Issue 14290 +func TestServerCloseClientConnections(t *testing.T) { + var s *Server + s = NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.CloseClientConnections() + })) + defer s.Close() + res, err := http.Get(s.URL) + if err == nil { + res.Body.Close() + t.Fatal("Unexpected response: %#v", res) + } +} diff --git a/libgo/go/net/http/httputil/reverseproxy.go b/libgo/go/net/http/httputil/reverseproxy.go index 4dba352..54411ca 100644 --- a/libgo/go/net/http/httputil/reverseproxy.go +++ b/libgo/go/net/http/httputil/reverseproxy.go @@ -106,11 +106,12 @@ func copyHeader(dst, src http.Header) { // http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html var hopHeaders = []string{ "Connection", + "Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", - "Te", // canonicalized version of "TE" - "Trailers", + "Te", // canonicalized version of "TE" + "Trailer", // not Trailers per URL above; http://www.rfc-editor.org/errata_search.php?eid=4522 "Transfer-Encoding", "Upgrade", } diff --git a/libgo/go/net/http/httputil/reverseproxy_test.go b/libgo/go/net/http/httputil/reverseproxy_test.go index 7f203d8..0849427 100644 --- a/libgo/go/net/http/httputil/reverseproxy_test.go +++ b/libgo/go/net/http/httputil/reverseproxy_test.go @@ -45,9 +45,13 @@ func TestReverseProxy(t *testing.T) { if c := r.Header.Get("Upgrade"); c != "" { t.Errorf("handler got Upgrade header value %q", c) } + if c := r.Header.Get("Proxy-Connection"); c != "" { + t.Errorf("handler got Proxy-Connection header value %q", c) + } if g, e := r.Host, "some-name"; g != e { t.Errorf("backend got Host header %q, want %q", g, e) } + w.Header().Set("Trailers", "not a special header field name") w.Header().Set("Trailer", "X-Trailer") w.Header().Set("X-Foo", "bar") w.Header().Set("Upgrade", "foo") @@ -71,6 +75,7 @@ func TestReverseProxy(t *testing.T) { getReq, _ := http.NewRequest("GET", frontend.URL, nil) getReq.Host = "some-name" getReq.Header.Set("Connection", "close") + getReq.Header.Set("Proxy-Connection", "should be deleted") getReq.Header.Set("Upgrade", "foo") getReq.Close = true res, err := http.DefaultClient.Do(getReq) @@ -86,6 +91,9 @@ func TestReverseProxy(t *testing.T) { if c := res.Header.Get(fakeHopHeader); c != "" { t.Errorf("got %s header value %q", fakeHopHeader, c) } + if g, e := res.Header.Get("Trailers"), "not a special header field name"; g != e { + t.Errorf("header Trailers = %q; want %q", g, e) + } if g, e := len(res.Header["X-Multi-Value"]), 2; g != e { t.Errorf("got %d X-Multi-Value header values; expected %d", g, e) } diff --git a/libgo/go/net/http/request.go b/libgo/go/net/http/request.go index 16c5bb4..8cdab02 100644 --- a/libgo/go/net/http/request.go +++ b/libgo/go/net/http/request.go @@ -99,30 +99,37 @@ type Request struct { ProtoMajor int // 1 ProtoMinor int // 0 - // A header maps request lines to their values. - // If the header says + // Header contains the request header fields either received + // by the server or to be sent by the client. // + // If a server received a request with header lines, + // + // Host: example.com // accept-encoding: gzip, deflate // Accept-Language: en-us - // Connection: keep-alive + // fOO: Bar + // foo: two // // then // // Header = map[string][]string{ // "Accept-Encoding": {"gzip, deflate"}, // "Accept-Language": {"en-us"}, - // "Connection": {"keep-alive"}, + // "Foo": {"Bar", "two"}, // } // - // HTTP defines that header names are case-insensitive. - // The request parser implements this by canonicalizing the - // name, making the first character and any characters - // following a hyphen uppercase and the rest lowercase. + // For incoming requests, the Host header is promoted to the + // Request.Host field and removed from the Header map. // - // For client requests certain headers are automatically - // added and may override values in Header. + // HTTP defines that header names are case-insensitive. The + // request parser implements this by using CanonicalHeaderKey, + // making the first character and any characters following a + // hyphen uppercase and the rest lowercase. // - // See the documentation for the Request.Write method. + // For client requests, certain headers such as Content-Length + // and Connection are automatically written when needed and + // values in Header may be ignored. See the documentation + // for the Request.Write method. Header Header // Body is the request's body. @@ -152,8 +159,15 @@ type Request struct { TransferEncoding []string // Close indicates whether to close the connection after - // replying to this request (for servers) or after sending - // the request (for clients). + // replying to this request (for servers) or after sending this + // request and reading its response (for clients). + // + // For server requests, the HTTP server handles this automatically + // and this field is not needed by Handlers. + // + // For client requests, setting this field prevents re-use of + // TCP connections between requests to the same hosts, as if + // Transport.DisableKeepAlives were set. Close bool // For server requests Host specifies the host on which the diff --git a/libgo/go/net/http/serve_test.go b/libgo/go/net/http/serve_test.go index f8cad80..384b453 100644 --- a/libgo/go/net/http/serve_test.go +++ b/libgo/go/net/http/serve_test.go @@ -1039,12 +1039,30 @@ func TestAutomaticHTTP2_Serve(t *testing.T) { } func TestAutomaticHTTP2_ListenAndServe(t *testing.T) { - defer afterTest(t) - defer SetTestHookServerServe(nil) cert, err := tls.X509KeyPair(internal.LocalhostCert, internal.LocalhostKey) if err != nil { t.Fatal(err) } + testAutomaticHTTP2_ListenAndServe(t, &tls.Config{ + Certificates: []tls.Certificate{cert}, + }) +} + +func TestAutomaticHTTP2_ListenAndServe_GetCertificate(t *testing.T) { + cert, err := tls.X509KeyPair(internal.LocalhostCert, internal.LocalhostKey) + if err != nil { + t.Fatal(err) + } + testAutomaticHTTP2_ListenAndServe(t, &tls.Config{ + GetCertificate: func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + return &cert, nil + }, + }) +} + +func testAutomaticHTTP2_ListenAndServe(t *testing.T, tlsConf *tls.Config) { + defer afterTest(t) + defer SetTestHookServerServe(nil) var ok bool var s *Server const maxTries = 5 @@ -1060,10 +1078,8 @@ Try: lnc <- ln }) s = &Server{ - Addr: addr, - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{cert}, - }, + Addr: addr, + TLSConfig: tlsConf, } errc := make(chan error, 1) go func() { errc <- s.ListenAndServeTLS("", "") }() @@ -2416,7 +2432,7 @@ func TestCloseNotifierPipelined(t *testing.T) { if err != nil { t.Fatalf("error dialing: %v", err) } - diec := make(chan bool, 2) + diec := make(chan bool, 1) go func() { const req = "GET / HTTP/1.1\r\nConnection: keep-alive\r\nHost: foo\r\n\r\n" _, err = io.WriteString(conn, req+req) // two requests @@ -2426,13 +2442,23 @@ func TestCloseNotifierPipelined(t *testing.T) { <-diec conn.Close() }() + reqs := 0 + closes := 0 For: for { select { case <-gotReq: - diec <- true + reqs++ + if reqs > 2 { + t.Fatal("too many requests") + } else if reqs > 1 { + diec <- true + } case <-sawClose: - break For + closes++ + if closes > 1 { + break For + } case <-time.After(5 * time.Second): ts.CloseClientConnections() t.Fatal("timeout") diff --git a/libgo/go/net/http/server.go b/libgo/go/net/http/server.go index 004a1f9..5e3b608 100644 --- a/libgo/go/net/http/server.go +++ b/libgo/go/net/http/server.go @@ -2233,10 +2233,11 @@ func ListenAndServeTLS(addr, certFile, keyFile string, handler Handler) error { // Accepted connections are configured to enable TCP keep-alives. // // Filenames containing a certificate and matching private key for the -// server must be provided if the Server's TLSConfig.Certificates is -// not populated. If the certificate is signed by a certificate -// authority, the certFile should be the concatenation of the server's -// certificate, any intermediates, and the CA's certificate. +// server must be provided if neither the Server's TLSConfig.Certificates +// nor TLSConfig.GetCertificate are populated. If the certificate is +// signed by a certificate authority, the certFile should be the +// concatenation of the server's certificate, any intermediates, and +// the CA's certificate. // // If srv.Addr is blank, ":https" is used. // @@ -2258,7 +2259,8 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { config.NextProtos = append(config.NextProtos, "http/1.1") } - if len(config.Certificates) == 0 || certFile != "" || keyFile != "" { + configHasCert := len(config.Certificates) > 0 || config.GetCertificate != nil + if !configHasCert || certFile != "" || keyFile != "" { var err error config.Certificates = make([]tls.Certificate, 1) config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) diff --git a/libgo/go/net/http/transport.go b/libgo/go/net/http/transport.go index 41df906..baf71d5 100644 --- a/libgo/go/net/http/transport.go +++ b/libgo/go/net/http/transport.go @@ -163,6 +163,22 @@ func (t *Transport) onceSetNextProtoDefaults() { return } if t.TLSNextProto != nil { + // This is the documented way to disable http2 on a + // Transport. + return + } + if t.TLSClientConfig != nil { + // Be conservative for now (for Go 1.6) at least and + // don't automatically enable http2 if they've + // specified a custom TLS config. Let them opt-in + // themselves via http2.ConfigureTransport so we don't + // surprise them by modifying their tls.Config. + // Issue 14275. + return + } + if t.ExpectContinueTimeout != 0 { + // Unsupported in http2, so disable http2 for now. + // Issue 13851. return } t2, err := http2configureTransport(t) diff --git a/libgo/go/net/http/transport_test.go b/libgo/go/net/http/transport_test.go index 3b2a5f9..0c901b3 100644 --- a/libgo/go/net/http/transport_test.go +++ b/libgo/go/net/http/transport_test.go @@ -2208,9 +2208,8 @@ func TestTransportTLSHandshakeTimeout(t *testing.T) { // Trying to repro golang.org/issue/3514 func TestTLSServerClosesConnection(t *testing.T) { defer afterTest(t) - if runtime.GOOS == "windows" { - t.Skip("skipping flaky test on Windows; golang.org/issue/7634") - } + setFlaky(t, 7634) + closedc := make(chan bool, 1) ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { if strings.Contains(r.URL.Path, "/keep-alive-then-die") { @@ -2886,23 +2885,34 @@ func TestTransportPrefersResponseOverWriteError(t *testing.T) { } func TestTransportAutomaticHTTP2(t *testing.T) { - tr := &Transport{} - _, err := tr.RoundTrip(new(Request)) - if err == nil { - t.Error("expected error from RoundTrip") - } - if tr.TLSNextProto["h2"] == nil { - t.Errorf("HTTP/2 not registered.") - } + testTransportAutoHTTP(t, &Transport{}, true) +} + +func TestTransportAutomaticHTTP2_TLSNextProto(t *testing.T) { + testTransportAutoHTTP(t, &Transport{ + TLSNextProto: make(map[string]func(string, *tls.Conn) RoundTripper), + }, false) +} + +func TestTransportAutomaticHTTP2_TLSConfig(t *testing.T) { + testTransportAutoHTTP(t, &Transport{ + TLSClientConfig: new(tls.Config), + }, false) +} + +func TestTransportAutomaticHTTP2_ExpectContinueTimeout(t *testing.T) { + testTransportAutoHTTP(t, &Transport{ + ExpectContinueTimeout: 1 * time.Second, + }, false) +} - // Now with TLSNextProto set: - tr = &Transport{TLSNextProto: make(map[string]func(string, *tls.Conn) RoundTripper)} - _, err = tr.RoundTrip(new(Request)) +func testTransportAutoHTTP(t *testing.T, tr *Transport, wantH2 bool) { + _, err := tr.RoundTrip(new(Request)) if err == nil { t.Error("expected error from RoundTrip") } - if tr.TLSNextProto["h2"] != nil { - t.Errorf("HTTP/2 registered, despite non-nil TLSNextProto field") + if reg := tr.TLSNextProto["h2"] != nil; reg != wantH2 { + t.Errorf("HTTP/2 registered = %v; want %v", reg, wantH2) } } diff --git a/libgo/go/net/net_test.go b/libgo/go/net/net_test.go index 6dcfc21..cd62b43 100644 --- a/libgo/go/net/net_test.go +++ b/libgo/go/net/net_test.go @@ -9,6 +9,7 @@ import ( "os" "runtime" "testing" + "time" ) func TestCloseRead(t *testing.T) { @@ -209,6 +210,7 @@ func TestListenerClose(t *testing.T) { defer os.Remove(ln.Addr().String()) } + dst := ln.Addr().String() if err := ln.Close(); err != nil { if perr := parseCloseError(err); perr != nil { t.Error(perr) @@ -222,9 +224,24 @@ func TestListenerClose(t *testing.T) { } if network == "tcp" { - cc, err := Dial("tcp", ln.Addr().String()) + // We will have two TCP FSMs inside the + // kernel here. There's no guarantee that a + // signal comes from the far end FSM will be + // delivered immediately to the near end FSM, + // especially on the platforms that allow + // multiple consumer threads to pull pending + // established connections at the same time by + // enabling SO_REUSEPORT option such as Linux, + // DragonFly BSD. So we need to give some time + // quantum to the kernel. + // + // Note that net.inet.tcp.reuseport_ext=1 by + // default on DragonFly BSD. + time.Sleep(time.Millisecond) + + cc, err := Dial("tcp", dst) if err == nil { - t.Error("Dial to closed TCP listener succeeeded.") + t.Error("Dial to closed TCP listener succeeded.") cc.Close() } } @@ -272,6 +289,9 @@ func TestListenCloseListen(t *testing.T) { } addr := ln.Addr().String() if err := ln.Close(); err != nil { + if perr := parseCloseError(err); perr != nil { + t.Error(perr) + } t.Fatal(err) } ln, err = Listen("tcp", addr) diff --git a/libgo/go/os/doc.go b/libgo/go/os/doc.go index 389a8eb..869a28a 100644 --- a/libgo/go/os/doc.go +++ b/libgo/go/os/doc.go @@ -7,9 +7,13 @@ package os import "time" // FindProcess looks for a running process by its pid. +// // The Process it returns can be used to obtain information // about the underlying operating system process. -func FindProcess(pid int) (p *Process, err error) { +// +// On Unix systems, FindProcess always succeeds and returns a Process +// for the given pid, regardless of whether the process exists. +func FindProcess(pid int) (*Process, error) { return findProcess(pid) } diff --git a/libgo/go/runtime/cgo_mmap.go b/libgo/go/runtime/cgo_mmap.go index ef5501c..c0396bd 100644 --- a/libgo/go/runtime/cgo_mmap.go +++ b/libgo/go/runtime/cgo_mmap.go @@ -15,12 +15,19 @@ import "unsafe" //go:linkname _cgo_mmap _cgo_mmap var _cgo_mmap unsafe.Pointer -func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (ret unsafe.Pointer) { +func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer { if _cgo_mmap != nil { + // Make ret a uintptr so that writing to it in the + // function literal does not trigger a write barrier. + // A write barrier here could break because of the way + // that mmap uses the same value both as a pointer and + // an errno value. + // TODO: Fix mmap to return two values. + var ret uintptr systemstack(func() { ret = callCgoMmap(addr, n, prot, flags, fd, off) }) - return + return unsafe.Pointer(ret) } return sysMmap(addr, n, prot, flags, fd, off) } @@ -31,4 +38,4 @@ func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) // cgoMmap calls the mmap function in the runtime/cgo package on the // callCgoMmap calls the mmap function in the runtime/cgo package // using the GCC calling convention. It is implemented in assembly. -func callCgoMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer +func callCgoMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) uintptr diff --git a/libgo/go/runtime/cgocheck.go b/libgo/go/runtime/cgocheck.go index 0077e22..aebce15 100644 --- a/libgo/go/runtime/cgocheck.go +++ b/libgo/go/runtime/cgocheck.go @@ -135,9 +135,6 @@ func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) { hbits := heapBitsForAddr(uintptr(src)) for i := uintptr(0); i < off+size; i += sys.PtrSize { bits := hbits.bits() - if bits != 0 { - println(i, bits) - } if i >= off && bits&bitPointer != 0 { v := *(*unsafe.Pointer)(add(src, i)) if cgoIsGoPointer(v) { diff --git a/libgo/go/runtime/crash_test.go b/libgo/go/runtime/crash_test.go index b622eb4..5f0e77b 100644 --- a/libgo/go/runtime/crash_test.go +++ b/libgo/go/runtime/crash_test.go @@ -317,3 +317,22 @@ func TestNetpollDeadlock(t *testing.T) { t.Fatalf("output does not start with %q:\n%s", want, output) } } + +func TestPanicTraceback(t *testing.T) { + output := runTestProg(t, "testprog", "PanicTraceback") + want := "panic: hello" + if !strings.HasPrefix(output, want) { + t.Fatalf("output does not start with %q:\n%s", want, output) + } + + // Check functions in the traceback. + fns := []string{"panic", "main.pt1.func1", "panic", "main.pt2.func1", "panic", "main.pt2", "main.pt1"} + for _, fn := range fns { + re := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(fn) + `\(.*\n`) + idx := re.FindStringIndex(output) + if idx == nil { + t.Fatalf("expected %q function in traceback:\n%s", fn, output) + } + output = output[idx[1]:] + } +} diff --git a/libgo/go/runtime/crash_unix_test.go b/libgo/go/runtime/crash_unix_test.go index 5284a37..771b303 100644 --- a/libgo/go/runtime/crash_unix_test.go +++ b/libgo/go/runtime/crash_unix_test.go @@ -14,6 +14,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strings" "syscall" "testing" ) @@ -52,6 +53,18 @@ func TestCrashDumpsAllThreads(t *testing.T) { cmd = exec.Command(filepath.Join(dir, "a.exe")) cmd = testEnv(cmd) cmd.Env = append(cmd.Env, "GOTRACEBACK=crash") + + // Set GOGC=off. Because of golang.org/issue/10958, the tight + // loops in the test program are not preemptible. If GC kicks + // in, it may lock up and prevent main from saying it's ready. + newEnv := []string{} + for _, s := range cmd.Env { + if !strings.HasPrefix(s, "GOGC=") { + newEnv = append(newEnv, s) + } + } + cmd.Env = append(newEnv, "GOGC=off") + var outbuf bytes.Buffer cmd.Stdout = &outbuf cmd.Stderr = &outbuf @@ -137,8 +150,8 @@ func loop(i int, c chan bool) { func TestSignalExitStatus(t *testing.T) { testenv.MustHaveGoBuild(t) switch runtime.GOOS { - case "netbsd": - t.Skip("skipping on NetBSD; see https://golang.org/issue/14063") + case "netbsd", "solaris": + t.Skipf("skipping on %s; see https://golang.org/issue/14063", runtime.GOOS) } exe, err := buildTestProg(t, "testprog") if err != nil { diff --git a/libgo/go/runtime/export_linux_test.go b/libgo/go/runtime/export_linux_test.go index 37cf164..4e76600 100644 --- a/libgo/go/runtime/export_linux_test.go +++ b/libgo/go/runtime/export_linux_test.go @@ -7,3 +7,4 @@ package runtime //var NewOSProc0 = newosproc0 +//var Mincore = mincore diff --git a/libgo/go/runtime/export_mmap_test.go b/libgo/go/runtime/export_mmap_test.go new file mode 100644 index 0000000..07b0a56 --- /dev/null +++ b/libgo/go/runtime/export_mmap_test.go @@ -0,0 +1,15 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Export guts for testing. + +package runtime + +var Mmap = mmap + +const ENOMEM = _ENOMEM +const MAP_ANON = _MAP_ANON +const MAP_PRIVATE = _MAP_PRIVATE diff --git a/libgo/go/runtime/pprof/pprof.go b/libgo/go/runtime/pprof/pprof.go index fa11fda..18e9936 100644 --- a/libgo/go/runtime/pprof/pprof.go +++ b/libgo/go/runtime/pprof/pprof.go @@ -351,7 +351,7 @@ func printStackRecord(w io.Writer, stk []uintptr, allFrames bool) { name := f.Name() // Hide runtime.goexit and any runtime functions at the beginning. // This is useful mainly for allocation traces. - wasPanic = name == "runtime.panic" + wasPanic = name == "runtime.gopanic" if name == "runtime.goexit" || !show && (strings.HasPrefix(name, "runtime.") || strings.HasPrefix(name, "runtime_")) { continue } diff --git a/libgo/go/runtime/pprof/pprof_test.go b/libgo/go/runtime/pprof/pprof_test.go index 244be05..e384e11 100644 --- a/libgo/go/runtime/pprof/pprof_test.go +++ b/libgo/go/runtime/pprof/pprof_test.go @@ -96,7 +96,7 @@ func parseProfile(t *testing.T, bytes []byte, f func(uintptr, []uintptr)) { if l < 5+3 { t.Logf("profile too short: %#x", val) if badOS[runtime.GOOS] { - t.Skipf("ignoring failure on %s; see golang.org/issue/6047", runtime.GOOS) + t.Skipf("ignoring failure on %s; see golang.org/issue/13841", runtime.GOOS) return } t.FailNow() @@ -171,7 +171,7 @@ func testCPUProfile(t *testing.T, need []string, f func(dur time.Duration)) { } if badOS[runtime.GOOS] { - t.Skipf("ignoring failure on %s; see golang.org/issue/6047", runtime.GOOS) + t.Skipf("ignoring failure on %s; see golang.org/issue/13841", runtime.GOOS) return } // Ignore the failure if the tests are running in a QEMU-based emulator, @@ -421,11 +421,13 @@ func deepStack(depth int) int { return deepStack(depth-1) + 1 } -// Operating systems that are expected to fail the tests. See issue 6047. +// Operating systems that are expected to fail the tests. See issue 13841. var badOS = map[string]bool{ - "darwin": true, - "netbsd": true, - "plan9": true, + "darwin": true, + "netbsd": true, + "plan9": true, + "dragonfly": true, + "solaris": true, } func TestBlockProfile(t *testing.T) { diff --git a/libgo/go/runtime/runtime-lldb_test.go b/libgo/go/runtime/runtime-lldb_test.go new file mode 100644 index 0000000..2bd91c1 --- /dev/null +++ b/libgo/go/runtime/runtime-lldb_test.go @@ -0,0 +1,262 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "debug/elf" + "debug/macho" + "encoding/binary" + "internal/testenv" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +var lldbPath string + +func checkLldbPython(t *testing.T) { + cmd := exec.Command("lldb", "-P") + out, err := cmd.CombinedOutput() + if err != nil { + t.Skipf("skipping due to issue running lldb: %v\n%s", err, out) + } + lldbPath = strings.TrimSpace(string(out)) + + cmd = exec.Command("/usr/bin/python2.7", "-c", "import sys;sys.path.append(sys.argv[1]);import lldb; print('go lldb python support')", lldbPath) + out, err = cmd.CombinedOutput() + + if err != nil { + t.Skipf("skipping due to issue running python: %v\n%s", err, out) + } + if string(out) != "go lldb python support\n" { + t.Skipf("skipping due to lack of python lldb support: %s", out) + } + + if runtime.GOOS == "darwin" { + // Try to see if we have debugging permissions. + cmd = exec.Command("/usr/sbin/DevToolsSecurity", "-status") + out, err = cmd.CombinedOutput() + if err != nil { + t.Skipf("DevToolsSecurity failed: %v", err) + } else if !strings.Contains(string(out), "enabled") { + t.Skip(string(out)) + } + cmd = exec.Command("/usr/bin/groups") + out, err = cmd.CombinedOutput() + if err != nil { + t.Skipf("groups failed: %v", err) + } else if !strings.Contains(string(out), "_developer") { + t.Skip("Not in _developer group") + } + } +} + +const lldbHelloSource = ` +package main +import "fmt" +func main() { + mapvar := make(map[string]string,5) + mapvar["abc"] = "def" + mapvar["ghi"] = "jkl" + intvar := 42 + ptrvar := &intvar + fmt.Println("hi") // line 10 + _ = ptrvar +} +` + +const lldbScriptSource = ` +import sys +sys.path.append(sys.argv[1]) +import lldb +import os + +TIMEOUT_SECS = 5 + +debugger = lldb.SBDebugger.Create() +debugger.SetAsync(True) +target = debugger.CreateTargetWithFileAndArch("a.exe", None) +if target: + print "Created target" + main_bp = target.BreakpointCreateByLocation("main.go", 10) + if main_bp: + print "Created breakpoint" + process = target.LaunchSimple(None, None, os.getcwd()) + if process: + print "Process launched" + listener = debugger.GetListener() + process.broadcaster.AddListener(listener, lldb.SBProcess.eBroadcastBitStateChanged) + while True: + event = lldb.SBEvent() + if listener.WaitForEvent(TIMEOUT_SECS, event): + if lldb.SBProcess.GetRestartedFromEvent(event): + continue + state = process.GetState() + if state in [lldb.eStateUnloaded, lldb.eStateLaunching, lldb.eStateRunning]: + continue + else: + print "Timeout launching" + break + if state == lldb.eStateStopped: + for t in process.threads: + if t.GetStopReason() == lldb.eStopReasonBreakpoint: + print "Hit breakpoint" + frame = t.GetFrameAtIndex(0) + if frame: + if frame.line_entry: + print "Stopped at %s:%d" % (frame.line_entry.file.basename, frame.line_entry.line) + if frame.function: + print "Stopped in %s" % (frame.function.name,) + var = frame.FindVariable('intvar') + if var: + print "intvar = %s" % (var.GetValue(),) + else: + print "no intvar" + else: + print "Process state", state + process.Destroy() +else: + print "Failed to create target a.exe" + +lldb.SBDebugger.Destroy(debugger) +sys.exit() +` + +const expectedLldbOutput = `Created target +Created breakpoint +Process launched +Hit breakpoint +Stopped at main.go:10 +Stopped in main.main +intvar = 42 +` + +func TestLldbPython(t *testing.T) { + testenv.MustHaveGoBuild(t) + if final := os.Getenv("GOROOT_FINAL"); final != "" && runtime.GOROOT() != final { + t.Skip("gdb test can fail with GOROOT_FINAL pending") + } + + checkLldbPython(t) + + dir, err := ioutil.TempDir("", "go-build") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(dir) + + src := filepath.Join(dir, "main.go") + err = ioutil.WriteFile(src, []byte(lldbHelloSource), 0644) + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + + cmd := exec.Command("go", "build", "-gcflags", "-N -l", "-o", "a.exe") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("building source %v\n%s", err, out) + } + + src = filepath.Join(dir, "script.py") + err = ioutil.WriteFile(src, []byte(lldbScriptSource), 0755) + if err != nil { + t.Fatalf("failed to create script: %v", err) + } + + cmd = exec.Command("/usr/bin/python2.7", "script.py", lldbPath) + cmd.Dir = dir + got, _ := cmd.CombinedOutput() + + if string(got) != expectedLldbOutput { + if strings.Contains(string(got), "Timeout launching") { + t.Skip("Timeout launching") + } + t.Fatalf("Unexpected lldb output:\n%s", got) + } +} + +// Check that aranges are valid even when lldb isn't installed. +func TestDwarfAranges(t *testing.T) { + testenv.MustHaveGoBuild(t) + dir, err := ioutil.TempDir("", "go-build") + if err != nil { + t.Fatalf("failed to create temp directory: %v", err) + } + defer os.RemoveAll(dir) + + src := filepath.Join(dir, "main.go") + err = ioutil.WriteFile(src, []byte(lldbHelloSource), 0644) + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + + cmd := exec.Command("go", "build", "-o", "a.exe") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("building source %v\n%s", err, out) + } + + filename := filepath.Join(dir, "a.exe") + if f, err := elf.Open(filename); err == nil { + sect := f.Section(".debug_aranges") + if sect == nil { + t.Fatal("Missing aranges section") + } + verifyAranges(t, f.ByteOrder, sect.Open()) + } else if f, err := macho.Open(filename); err == nil { + sect := f.Section("__debug_aranges") + if sect == nil { + t.Fatal("Missing aranges section") + } + verifyAranges(t, f.ByteOrder, sect.Open()) + } else { + t.Skip("Not an elf or macho binary.") + } +} + +func verifyAranges(t *testing.T, byteorder binary.ByteOrder, data io.ReadSeeker) { + var header struct { + UnitLength uint32 // does not include the UnitLength field + Version uint16 + Offset uint32 + AddressSize uint8 + SegmentSize uint8 + } + for { + offset, err := data.Seek(0, 1) + if err != nil { + t.Fatalf("Seek error: %v", err) + } + if err = binary.Read(data, byteorder, &header); err == io.EOF { + return + } else if err != nil { + t.Fatalf("Error reading arange header: %v", err) + } + tupleSize := int64(header.SegmentSize) + 2*int64(header.AddressSize) + lastTupleOffset := offset + int64(header.UnitLength) + 4 - tupleSize + if lastTupleOffset%tupleSize != 0 { + t.Fatalf("Invalid arange length %d, (addr %d, seg %d)", header.UnitLength, header.AddressSize, header.SegmentSize) + } + if _, err = data.Seek(lastTupleOffset, 0); err != nil { + t.Fatalf("Seek error: %v", err) + } + buf := make([]byte, tupleSize) + if n, err := data.Read(buf); err != nil || int64(n) < tupleSize { + t.Fatalf("Read error: %v", err) + } + for _, val := range buf { + if val != 0 { + t.Fatalf("Invalid terminator") + } + } + } +} diff --git a/libgo/go/runtime/runtime_mmap_test.go b/libgo/go/runtime/runtime_mmap_test.go new file mode 100644 index 0000000..3995305 --- /dev/null +++ b/libgo/go/runtime/runtime_mmap_test.go @@ -0,0 +1,30 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package runtime_test + +import ( + "runtime" + "runtime/internal/sys" + "testing" +) + +// Test that the error value returned by mmap is positive, as that is +// what the code in mem_bsd.go, mem_darwin.go, and mem_linux.go expects. +// See the uses of ENOMEM in sysMap in those files. +func TestMmapErrorSign(t *testing.T) { + p := runtime.Mmap(nil, ^uintptr(0)&^(sys.PhysPageSize-1), 0, runtime.MAP_ANON|runtime.MAP_PRIVATE, -1, 0) + + // The runtime.mmap function is nosplit, but t.Errorf is not. + // Reset the pointer so that we don't get an "invalid stack + // pointer" error from t.Errorf if we call it. + v := uintptr(p) + p = nil + + if v != runtime.ENOMEM { + t.Errorf("mmap = %v, want %v", v, runtime.ENOMEM) + } +} diff --git a/libgo/go/runtime/runtime_test.go b/libgo/go/runtime/runtime_test.go index 980a9f8..a520f56 100644 --- a/libgo/go/runtime/runtime_test.go +++ b/libgo/go/runtime/runtime_test.go @@ -312,13 +312,22 @@ func TestAppendSliceGrowth(t *testing.T) { } func TestGoroutineProfileTrivial(t *testing.T) { - n1, ok := GoroutineProfile(nil) // should fail, there's at least 1 goroutine - if n1 < 1 || ok { - t.Fatalf("GoroutineProfile(nil) = %d, %v, want >0, false", n1, ok) - } - - n2, ok := GoroutineProfile(make([]StackRecord, n1)) - if n2 != n1 || !ok { - t.Fatalf("GoroutineProfile(%d) = %d, %v, want %d, true", n1, n2, ok, n1) + // Calling GoroutineProfile twice in a row should find the same number of goroutines, + // but it's possible there are goroutines just about to exit, so we might end up + // with fewer in the second call. Try a few times; it should converge once those + // zombies are gone. + for i := 0; ; i++ { + n1, ok := GoroutineProfile(nil) // should fail, there's at least 1 goroutine + if n1 < 1 || ok { + t.Fatalf("GoroutineProfile(nil) = %d, %v, want >0, false", n1, ok) + } + n2, ok := GoroutineProfile(make([]StackRecord, n1)) + if n2 == n1 && ok { + break + } + t.Logf("GoroutineProfile(%d) = %d, %v, want %d, true", n1, n2, ok, n1) + if i >= 10 { + t.Fatalf("GoroutineProfile not converging") + } } } diff --git a/libgo/go/runtime/string_test.go b/libgo/go/runtime/string_test.go index e0967b3..64abd57 100644 --- a/libgo/go/runtime/string_test.go +++ b/libgo/go/runtime/string_test.go @@ -225,3 +225,18 @@ func TestRangeStringCast(t *testing.T) { t.Fatalf("want 0 allocs, got %v", n) } } + +func TestString2Slice(t *testing.T) { + // Make sure we don't return slices that expose + // an unzeroed section of stack-allocated temp buf + // between len and cap. See issue 14232. + s := "foož" + b := ([]byte)(s) + if cap(b) != 5 { + t.Errorf("want cap of 5, got %d", cap(b)) + } + r := ([]rune)(s) + if cap(r) != 4 { + t.Errorf("want cap of 4, got %d", cap(r)) + } +} diff --git a/libgo/go/runtime/testdata/testprog/deadlock.go b/libgo/go/runtime/testdata/testprog/deadlock.go index 7f0a0cd..73fbf62 100644 --- a/libgo/go/runtime/testdata/testprog/deadlock.go +++ b/libgo/go/runtime/testdata/testprog/deadlock.go @@ -29,7 +29,7 @@ func init() { register("GoexitInPanic", GoexitInPanic) register("PanicAfterGoexit", PanicAfterGoexit) register("RecoveredPanicAfterGoexit", RecoveredPanicAfterGoexit) - + register("PanicTraceback", PanicTraceback) } func SimpleDeadlock() { @@ -171,3 +171,21 @@ func RecoveredPanicAfterGoexit() { }() runtime.Goexit() } + +func PanicTraceback() { + pt1() +} + +func pt1() { + defer func() { + panic("panic pt1") + }() + pt2() +} + +func pt2() { + defer func() { + panic("panic pt2") + }() + panic("hello") +} diff --git a/libgo/go/sync/waitgroup_test.go b/libgo/go/sync/waitgroup_test.go index a581660..8ec34fd 100644 --- a/libgo/go/sync/waitgroup_test.go +++ b/libgo/go/sync/waitgroup_test.go @@ -128,13 +128,16 @@ func TestWaitGroupMisuse3(t *testing.T) { } }() defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) - done := make(chan interface{}, 1) + done := make(chan interface{}, 2) // The detection is opportunistically, so we want it to panic // at least in one run out of a million. for i := 0; i < 1e6; i++ { var wg WaitGroup wg.Add(1) go func() { + defer func() { + done <- recover() + }() wg.Done() }() go func() { @@ -150,8 +153,10 @@ func TestWaitGroupMisuse3(t *testing.T) { wg.Wait() }() wg.Wait() - if err := <-done; err != nil { - panic(err) + for j := 0; j < 2; j++ { + if err := <-done; err != nil { + panic(err) + } } } t.Fatal("Should panic") diff --git a/libgo/runtime/go-string-to-byte-array.c b/libgo/runtime/go-string-to-byte-array.c index a4edb50..61591eb 100644 --- a/libgo/runtime/go-string-to-byte-array.c +++ b/libgo/runtime/go-string-to-byte-array.c @@ -23,6 +23,6 @@ __go_string_to_byte_array (String str) __builtin_memset (data + str.len, 0, cap - (uintptr) str.len); ret.__values = (void *) data; ret.__count = str.len; - ret.__capacity = (intgo) cap; + ret.__capacity = str.len; return ret; }