From a5b8971c8de356c83d6f737dd9a4508e11916052 Mon Sep 17 00:00:00 2001 From: Krzysztof Wicher Date: Fri, 7 Jun 2019 14:38:02 -0700 Subject: [PATCH] HttpClient: Try decode Location header using UTF-8 (dotnet/corefx#37852) * Use desktop logic for parsing Location (try decode using UTF-8) * Tests and feedback * fix netfx * Convert UTF-8 in non-allocating way without try .. catch Commit migrated from https://github.com/dotnet/corefx/commit/02026907370c57380a9f091b54ee77873f276a68 --- .../Common/tests/System/Net/Http/LoopbackServer.cs | 21 +++++++++ .../System/Net/Http/Headers/HeaderDescriptor.cs | 45 ++++++++++++++++-- .../FunctionalTests/SocketsHttpHandlerTest.cs | 54 ++++++++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/src/libraries/Common/tests/System/Net/Http/LoopbackServer.cs b/src/libraries/Common/tests/System/Net/Http/LoopbackServer.cs index b6a20b4..d826d24 100644 --- a/src/libraries/Common/tests/System/Net/Http/LoopbackServer.cs +++ b/src/libraries/Common/tests/System/Net/Http/LoopbackServer.cs @@ -122,6 +122,20 @@ namespace System.Net.Test.Common { List lines = null; + // Note, we assume there's no request body. + // We'll close the connection after reading the request header and sending the response. + await AcceptConnectionAsync(async connection => + { + lines = await connection.ReadRequestHeaderAndSendCustomResponseAsync(response).ConfigureAwait(false); + }); + + return lines; + } + + public async Task> AcceptConnectionSendCustomResponseAndCloseAsync(byte[] response) + { + List lines = null; + // Note, we assume there's no request body. // We'll close the connection after reading the request header and sending the response. await AcceptConnectionAsync(async connection => @@ -591,6 +605,13 @@ namespace System.Net.Test.Common return lines; } + public async Task> ReadRequestHeaderAndSendCustomResponseAsync(byte[] response) + { + List lines = await ReadRequestHeaderAsync().ConfigureAwait(false); + await _stream.WriteAsync(response, 0, response.Length).ConfigureAwait(false); + return lines; + } + public async Task> ReadRequestHeaderAndSendResponseAsync(HttpStatusCode statusCode = HttpStatusCode.OK, string additionalHeaders = null, string content = null) { List lines = await ReadRequestHeaderAsync().ConfigureAwait(false); diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/Headers/HeaderDescriptor.cs b/src/libraries/System.Net.Http/src/System/Net/Http/Headers/HeaderDescriptor.cs index e469e71..ef2443a 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/Headers/HeaderDescriptor.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/Headers/HeaderDescriptor.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Buffers; using System.Diagnostics; +using System.Text.Unicode; namespace System.Net.Http.Headers { @@ -99,19 +101,52 @@ namespace System.Net.Http.Headers } // If it's a known header value, use the known value instead of allocating a new string. - if (_knownHeader != null && _knownHeader.KnownValues != null) + if (_knownHeader != null) { - string[] knownValues = _knownHeader.KnownValues; - for (int i = 0; i < knownValues.Length; i++) + if (_knownHeader.KnownValues != null) { - if (ByteArrayHelpers.EqualsOrdinalAsciiIgnoreCase(knownValues[i], headerValue)) + string[] knownValues = _knownHeader.KnownValues; + for (int i = 0; i < knownValues.Length; i++) { - return knownValues[i]; + if (ByteArrayHelpers.EqualsOrdinalAsciiIgnoreCase(knownValues[i], headerValue)) + { + return knownValues[i]; + } + } + } + + if (_knownHeader == KnownHeaders.Location) + { + // Normally Location should be in ISO-8859-1 but occasionally some servers respond with UTF-8. + if (TryDecodeUtf8(headerValue, out string decoded)) + { + return decoded; } } } return HttpRuleParser.DefaultHttpEncoding.GetString(headerValue); } + + private static bool TryDecodeUtf8(ReadOnlySpan input, out string decoded) + { + char[] rented = ArrayPool.Shared.Rent(input.Length); + + try + { + if (Utf8.ToUtf16(input, rented, out _, out int charsWritten, replaceInvalidSequences: false) == OperationStatus.Done) + { + decoded = new string(rented, 0, charsWritten); + return true; + } + } + finally + { + ArrayPool.Shared.Return(rented); + } + + decoded = null; + return false; + } } } diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs index 92c0231..92cb7ed 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs @@ -2166,6 +2166,60 @@ namespace System.Net.Http.Functional.Tests } } + public sealed class SocketsHttpHandlerTest_LocationHeader + { + private static readonly byte[] s_redirectResponseBefore = Encoding.ASCII.GetBytes( + "HTTP/1.1 301 Moved Permanently\r\n" + + "Connection: close\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Location: "); + + private static readonly byte[] s_redirectResponseAfter = Encoding.ASCII.GetBytes( + "\r\n" + + "Server: Loopback\r\n" + + "\r\n" + + "0\r\n\r\n"); + + [Theory] + // US-ASCII only + [InlineData("http://a/", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/' })] + [InlineData("http://a/asdasd", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/', (byte)'a', (byte)'s', (byte)'d', (byte)'a', (byte)'s', (byte)'d' })] + // 2, 3, 4 byte UTF-8 characters + [InlineData("http://a/\u00A2", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/', 0xC2, 0xA2 })] + [InlineData("http://a/\u20AC", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/', 0xE2, 0x82, 0xAC })] + [InlineData("http://a/\uD800\uDF48", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/', 0xF0, 0x90, 0x8D, 0x88 })] + // 3 Polish letters + [InlineData("http://a/\u0105\u015B\u0107", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/', 0xC4, 0x85, 0xC5, 0x9B, 0xC4, 0x87 })] + // Negative cases - should be interpreted as ISO-8859-1 + // Invalid utf-8 sequence (continuation without start) + [InlineData("http://a/%C2%80", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/', 0b10000000 })] + // Invalid utf-8 sequence (not allowed character) + [InlineData("http://a/\u00C3\u0028", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/', 0xC3, 0x28 })] + // Incomplete utf-8 sequence + [InlineData("http://a/\u00C2", new byte[] { (byte)'h', (byte)'t', (byte)'t', (byte)'p', (byte)':', (byte)'/', (byte)'/', (byte)'a', (byte)'/', 0xC2 })] + public async void LocationHeader_DecodesUtf8_Success(string expected, byte[] location) + { + await LoopbackServer.CreateClientAndServerAsync(async url => + { + using (HttpClientHandler handler = new HttpClientHandler()) + { + handler.AllowAutoRedirect = false; + + using (HttpClient client = new HttpClient(handler)) + { + HttpResponseMessage response = await client.GetAsync(url); + Assert.Equal(expected, response.Headers.Location.ToString()); + } + } + }, server => server.AcceptConnectionSendCustomResponseAndCloseAsync(PreperateResponseWithRedirect(location))); + } + + private static byte[] PreperateResponseWithRedirect(byte[] location) + { + return s_redirectResponseBefore.Concat(location).Concat(s_redirectResponseAfter).ToArray(); + } + } + public sealed class SocketsHttpHandlerTest_Http2 : HttpClientHandlerTest_Http2 { public SocketsHttpHandlerTest_Http2(ITestOutputHelper output) : base(output) { } -- 2.7.4