From 4484947e53370d281c2c224f27855af8a40c3099 Mon Sep 17 00:00:00 2001 From: David Shulman Date: Thu, 11 Apr 2019 17:31:25 -0700 Subject: [PATCH] Improve HttpClient POST tests stability (dotnet/corefx#36801) Several of the HttpClient POST scenario tests were failing with HTTP status code 500. This was caused by the Azure remote endpoint generating OutOfMemoryException at times. The tests are sending large amounts of request body data to the Echo endpoint. That endpoint will respond back with all the headers and request body data serialized into a JSON payload. The OOM exceptions were coming from the Newtonsoft JSON serialization code currently used by the server endpoint. These tests don't really require that the request body data be sent back. The purpose of the tests is to verify that the request body payload was correctly sent by the client. We already have an endpoint, VerifyUpload, that can do that without echo'ing back the large request body data. This PR modifies the tests to use that endpoint. This should mitigate the OOM exceptions currently being generated in the server-side code. Additional mitigations/fixes will be done later on the server-side code to improve robustness. But fixing the tests to be more streamlined is goodness and will result in these tests being faster and more stable. Closes dotnet/corefx#36782 Commit migrated from https://github.com/dotnet/corefx/commit/288ce93787799052705814876d6e5ccb0523797d --- .../Common/tests/System/Net/Configuration.Http.cs | 1 + .../tests/FunctionalTests/PostScenarioTest.cs | 46 +++++++++------------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/src/libraries/Common/tests/System/Net/Configuration.Http.cs b/src/libraries/Common/tests/System/Net/Configuration.Http.cs index 178a332..1c388fe 100644 --- a/src/libraries/Common/tests/System/Net/Configuration.Http.cs +++ b/src/libraries/Common/tests/System/Net/Configuration.Http.cs @@ -61,6 +61,7 @@ namespace System.Net.Test.Common public static readonly Uri RemoteVerifyUploadServer = new Uri("http://" + Host + "/" + VerifyUploadHandler); public static readonly Uri SecureRemoteVerifyUploadServer = new Uri("https://" + SecureHost + "/" + VerifyUploadHandler); public static readonly Uri Http2RemoteVerifyUploadServer = new Uri("https://" + Http2Host + "/" + VerifyUploadHandler); + public static readonly Uri[] VerifyUploadServerList = new Uri[] { RemoteVerifyUploadServer, SecureRemoteVerifyUploadServer, Http2RemoteVerifyUploadServer }; public static readonly Uri RemoteEmptyContentServer = new Uri("http://" + Host + "/" + EmptyContentHandler); public static readonly Uri RemoteDeflateServer = new Uri("http://" + Host + "/" + DeflateHandler); diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/PostScenarioTest.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/PostScenarioTest.cs index 63f23cf..2ff7d2a 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/PostScenarioTest.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/PostScenarioTest.cs @@ -28,6 +28,7 @@ namespace System.Net.Http.Functional.Tests private readonly ITestOutputHelper _output; public static readonly object[][] EchoServers = Configuration.Http.EchoServers; + public static readonly object[][] VerifyUploadServers = Configuration.Http.VerifyUploadServers; public static readonly object[][] BasicAuthEchoServers = new object[][] @@ -67,7 +68,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostNoContentUsingContentLengthSemantics_Success(Uri serverUri) { await PostHelper(serverUri, string.Empty, null, @@ -75,7 +76,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostEmptyContentUsingContentLengthSemantics_Success(Uri serverUri) { await PostHelper(serverUri, string.Empty, new StringContent(string.Empty), @@ -83,7 +84,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostEmptyContentUsingChunkedEncoding_Success(Uri serverUri) { await PostHelper(serverUri, string.Empty, new StringContent(string.Empty), @@ -91,7 +92,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostEmptyContentUsingConflictingSemantics_Success(Uri serverUri) { await PostHelper(serverUri, string.Empty, new StringContent(string.Empty), @@ -99,7 +100,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostUsingContentLengthSemantics_Success(Uri serverUri) { await PostHelper(serverUri, ExpectedContent, new StringContent(ExpectedContent), @@ -107,7 +108,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostUsingChunkedEncoding_Success(Uri serverUri) { await PostHelper(serverUri, ExpectedContent, new StringContent(ExpectedContent), @@ -115,7 +116,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostSyncBlockingContentUsingChunkedEncoding_Success(Uri serverUri) { await PostHelper(serverUri, ExpectedContent, new SyncBlockingContent(ExpectedContent), @@ -123,7 +124,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostRepeatedFlushContentUsingChunkedEncoding_Success(Uri serverUri) { await PostHelper(serverUri, ExpectedContent, new RepeatedFlushContent(ExpectedContent), @@ -131,7 +132,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] public async Task PostUsingUsingConflictingSemantics_UsesChunkedSemantics(Uri serverUri) { await PostHelper(serverUri, ExpectedContent, new StringContent(ExpectedContent), @@ -139,7 +140,7 @@ namespace System.Net.Http.Functional.Tests } [OuterLoop("Uses external servers")] - [Theory, MemberData(nameof(EchoServers))] + [Theory, MemberData(nameof(VerifyUploadServers))] [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, "netfx behaves differently and will buffer content and use 'Content-Length' semantics")] [SkipOnTargetFramework(TargetFrameworkMonikers.Uap, "WinRT behaves differently and will use 'Content-Length' semantics")] public async Task PostUsingNoSpecifiedSemantics_UsesChunkedSemantics(Uri serverUri) @@ -148,9 +149,9 @@ namespace System.Net.Http.Functional.Tests useContentLengthUpload: false, useChunkedEncodingUpload: false); } - public static IEnumerable EchoServersAndLargeContentSizes() + public static IEnumerable VerifyUploadServersAndLargeContentSizes() { - foreach (Uri uri in Configuration.Http.EchoServerList) + foreach (Uri uri in Configuration.Http.VerifyUploadServerList) { yield return new object[] { uri, 5 * 1024 }; yield return new object[] { uri, 63 * 1024 }; @@ -160,7 +161,7 @@ namespace System.Net.Http.Functional.Tests [OuterLoop("Uses external server")] [Theory] - [MemberData(nameof(EchoServersAndLargeContentSizes))] + [MemberData(nameof(VerifyUploadServersAndLargeContentSizes))] public async Task PostLargeContentUsingContentLengthSemantics_Success(Uri serverUri, int contentLength) { var rand = new Random(42); @@ -246,8 +247,12 @@ namespace System.Net.Http.Functional.Tests { requestContent.Headers.ContentLength = null; } + + // Compute MD5 of request body data. This will be verified by the server when it + // receives the request. + requestContent.Headers.ContentMD5 = TestHelper.ComputeMD5Hash(requestBody); } - + if (useChunkedEncodingUpload) { client.DefaultRequestHeaders.TransferEncodingChunked = true; @@ -256,19 +261,6 @@ namespace System.Net.Http.Functional.Tests using (HttpResponseMessage response = await client.PostAsync(serverUri, requestContent)) { Assert.Equal(HttpStatusCode.OK, response.StatusCode); - string responseContent = await response.Content.ReadAsStringAsync(); - _output.WriteLine(responseContent); - - if (!useContentLengthUpload && !useChunkedEncodingUpload) - { - useChunkedEncodingUpload = true; - } - - TestHelper.VerifyResponseBody( - responseContent, - response.Content.Headers.ContentMD5, - useChunkedEncodingUpload, - requestBody); } } } -- 2.7.4