From d3beb6014a14bcb6824239e6db51f9505452586c Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 9 Oct 2020 06:12:47 -0400 Subject: [PATCH] Implement ZLibStream and fix SocketsHttpHandler deflate support (#42717) * Implement ZLibStream and fix SocketsHttpHandler deflate support - Implements ZLibStream, exposes it in the ref, and add tests - Fixes SocketsHttpHandler to use ZLibStream instead of DeflateStream * Add comment about deflate content encoding * Apply suggestions from code review Co-authored-by: Carlos Sanchez <1175054+carlossanlop@users.noreply.github.com> * Fix netfx build Co-authored-by: Carlos Sanchez <1175054+carlossanlop@users.noreply.github.com> --- .../Compression/CompressionStreamUnitTestBase.cs | 53 ++++ .../Http/HttpClientHandlerTest.Decompression.cs | 101 ++++--- .../tests/CompressionStreamUnitTests.Brotli.cs | 36 --- .../ref/System.IO.Compression.cs | 35 +++ .../src/System.IO.Compression.csproj | 3 +- .../DeflateManaged/DeflateManagedStream.cs | 2 +- .../IO/Compression/DeflateZLib/DeflateStream.cs | 2 +- .../IO/Compression/DeflateZLib/ZLibNative.cs | 8 + .../src/System/IO/Compression/GZipStream.cs | 2 +- .../src/System/IO/Compression/ZLibStream.cs | 293 +++++++++++++++++++++ .../tests/CompressionStreamUnitTests.Deflate.cs | 83 ++---- .../tests/CompressionStreamUnitTests.Gzip.cs | 64 +---- .../tests/CompressionStreamUnitTests.ZLib.cs | 20 ++ .../tests/System.IO.Compression.Tests.csproj | 31 +-- .../tests/UnitTests/TestServer.cs | 14 +- .../SocketsHttpHandler/DecompressionHandler.cs | 7 +- 16 files changed, 520 insertions(+), 234 deletions(-) create mode 100644 src/libraries/System.IO.Compression/src/System/IO/Compression/ZLibStream.cs create mode 100644 src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs diff --git a/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs b/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs index 90a6c35..96db35c 100644 --- a/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs +++ b/src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs @@ -1254,6 +1254,59 @@ namespace System.IO.Compression Assert.Equal(sourceData, decompressedStream.ToArray()); }))); } + + [Fact] + public void Precancellation() + { + var ms = new MemoryStream(); + using (Stream compressor = CreateStream(ms, CompressionMode.Compress, leaveOpen: true)) + { + Assert.True(compressor.WriteAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled); + Assert.True(compressor.FlushAsync(new CancellationToken(true)).IsCanceled); + } + using (Stream decompressor = CreateStream(ms, CompressionMode.Decompress, leaveOpen: true)) + { + Assert.True(decompressor.ReadAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisposeAsync_Flushes(bool leaveOpen) + { + var ms = new MemoryStream(); + var cs = CreateStream(ms, CompressionMode.Compress, leaveOpen); + cs.WriteByte(1); + await cs.FlushAsync(); + + long pos = ms.Position; + cs.WriteByte(1); + Assert.Equal(pos, ms.Position); + + await cs.DisposeAsync(); + Assert.InRange(ms.ToArray().Length, pos + 1, int.MaxValue); + if (leaveOpen) + { + Assert.InRange(ms.Position, pos + 1, int.MaxValue); + } + else + { + Assert.Throws(() => ms.Position); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisposeAsync_MultipleCallsAllowed(bool leaveOpen) + { + using (var cs = CreateStream(new MemoryStream(), CompressionMode.Compress, leaveOpen)) + { + await cs.DisposeAsync(); + await cs.DisposeAsync(); + } + } } internal sealed class BadWrappedStream : MemoryStream diff --git a/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Decompression.cs b/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Decompression.cs index 4877f75..676845a 100644 --- a/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Decompression.cs +++ b/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Decompression.cs @@ -33,46 +33,55 @@ namespace System.Net.Http.Functional.Tests foreach (Configuration.Http.RemoteServer remoteServer in Configuration.Http.RemoteServers) { yield return new object[] { remoteServer, remoteServer.GZipUri }; - yield return new object[] { remoteServer, remoteServer.DeflateUri }; + + // Remote deflate endpoint isn't correctly following the deflate protocol. + //yield return new object[] { remoteServer, remoteServer.DeflateUri }; } } - public static IEnumerable DecompressedResponse_MethodSpecified_DecompressedContentReturned_MemberData() + [Theory] + [InlineData("gzip", false)] + [InlineData("gzip", true)] + [InlineData("deflate", false)] + [InlineData("deflate", true)] + [InlineData("br", false)] + [InlineData("br", true)] + public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturned(string encodingName, bool all) { - foreach (bool specifyAllMethods in new[] { false, true }) + Func compress; + DecompressionMethods methods; + switch (encodingName) { - yield return new object[] - { - "deflate", - new Func(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)), - specifyAllMethods ? DecompressionMethods.Deflate : _all - }; - yield return new object[] - { - "gzip", - new Func(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)), - specifyAllMethods ? DecompressionMethods.GZip : _all - }; + case "gzip": + compress = s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true); + methods = all ? DecompressionMethods.GZip : _all; + break; + #if !NETFRAMEWORK - yield return new object[] - { - "br", - new Func(s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true)), - specifyAllMethods ? DecompressionMethods.Brotli : _all - }; + case "br": + if (IsWinHttpHandler) + { + // Brotli only supported on SocketsHttpHandler. + return; + } + + compress = s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true); + methods = all ? DecompressionMethods.Brotli : _all; + break; + + case "deflate": + // WinHttpHandler continues to use DeflateStream as it doesn't have a newer build than netstandard2.0 + // and doesn't have access to ZLibStream. + compress = IsWinHttpHandler ? + new Func(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)) : + new Func(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true)); + methods = all ? DecompressionMethods.Deflate : _all; + break; #endif - } - } - [Theory] - [MemberData(nameof(DecompressedResponse_MethodSpecified_DecompressedContentReturned_MemberData))] - public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturned( - string encodingName, Func compress, DecompressionMethods methods) - { - // Brotli only supported on SocketsHttpHandler. - if (IsWinHttpHandler && encodingName == "br") - { - return; + default: + Assert.Contains(encodingName, new[] { "br", "deflate", "gzip" }); + return; } var expectedContent = new byte[12345]; @@ -104,15 +113,15 @@ namespace System.Net.Http.Functional.Tests { yield return new object[] { - "deflate", - new Func(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)), + "gzip", + new Func(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)), DecompressionMethods.None }; #if !NETFRAMEWORK yield return new object[] { - "gzip", - new Func(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)), + "deflate", + new Func(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true)), DecompressionMethods.Brotli }; yield return new object[] @@ -186,6 +195,26 @@ namespace System.Net.Http.Functional.Tests } } + // The remote server endpoint was written to use DeflateStream, which isn't actually a correct + // implementation of the deflate protocol (the deflate protocol requires the zlib wrapper around + // deflate). Until we can get that updated (and deal with previous releases still testing it + // via a DeflateStream-based implementation), we utilize httpbin.org to help validate behavior. + [OuterLoop("Uses external servers")] + [Theory] + [InlineData("http://httpbin.org/deflate", "\"deflated\": true")] + [InlineData("https://httpbin.org/deflate", "\"deflated\": true")] + [InlineData("http://httpbin.org/gzip", "\"gzipped\": true")] + [InlineData("https://httpbin.org/gzip", "\"gzipped\": true")] + public async Task GetAsync_SetAutomaticDecompression_ContentDecompressed(string uri, string expectedContent) + { + HttpClientHandler handler = CreateHttpClientHandler(); + handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + using (HttpClient client = CreateHttpClient(handler)) + { + Assert.Contains(expectedContent, await client.GetStringAsync(uri)); + } + } + [OuterLoop("Uses external server")] [Theory, MemberData(nameof(RemoteServersAndCompressionUris))] public async Task GetAsync_SetAutomaticDecompression_HeadersRemoved(Configuration.Http.RemoteServer remoteServer, Uri uri) diff --git a/src/libraries/System.IO.Compression.Brotli/tests/CompressionStreamUnitTests.Brotli.cs b/src/libraries/System.IO.Compression.Brotli/tests/CompressionStreamUnitTests.Brotli.cs index c4c2f37..f1c3b00 100644 --- a/src/libraries/System.IO.Compression.Brotli/tests/CompressionStreamUnitTests.Brotli.cs +++ b/src/libraries/System.IO.Compression.Brotli/tests/CompressionStreamUnitTests.Brotli.cs @@ -24,42 +24,6 @@ namespace System.IO.Compression protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("BrotliTestData", Path.GetFileName(uncompressedPath) + ".br"); [Fact] - public void Precancellation() - { - var ms = new MemoryStream(); - using (Stream compressor = new BrotliStream(ms, CompressionMode.Compress, leaveOpen: true)) - { - Assert.True(compressor.WriteAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled); - Assert.True(compressor.FlushAsync(new CancellationToken(true)).IsCanceled); - } - using (Stream decompressor = CreateStream(ms, CompressionMode.Decompress, leaveOpen: true)) - { - Assert.True(decompressor.ReadAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled); - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task DisposeAsync_Flushes(bool leaveOpen) - { - var ms = new MemoryStream(); - var bs = new BrotliStream(ms, CompressionMode.Compress, leaveOpen); - bs.WriteByte(1); - Assert.Equal(0, ms.Position); - await bs.DisposeAsync(); - Assert.InRange(ms.ToArray().Length, 1, int.MaxValue); - if (leaveOpen) - { - Assert.InRange(ms.Position, 1, int.MaxValue); - } - else - { - Assert.Throws(() => ms.Position); - } - } - - [Fact] [OuterLoop("Test takes ~6 seconds to run")] public override void FlushAsync_DuringWriteAsync() { base.FlushAsync_DuringWriteAsync(); } diff --git a/src/libraries/System.IO.Compression/ref/System.IO.Compression.cs b/src/libraries/System.IO.Compression/ref/System.IO.Compression.cs index 11093ab..0a8d963 100644 --- a/src/libraries/System.IO.Compression/ref/System.IO.Compression.cs +++ b/src/libraries/System.IO.Compression/ref/System.IO.Compression.cs @@ -121,4 +121,39 @@ namespace System.IO.Compression Create = 1, Update = 2, } + public sealed partial class ZLibStream : System.IO.Stream + { + public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionLevel compressionLevel) { } + public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionLevel compressionLevel, bool leaveOpen) { } + public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionMode mode) { } + public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionMode mode, bool leaveOpen) { } + public System.IO.Stream BaseStream { get { throw null; } } + public override bool CanRead { get { throw null; } } + public override bool CanSeek { get { throw null; } } + public override bool CanWrite { get { throw null; } } + public override long Length { get { throw null; } } + public override long Position { get { throw null; } set { } } + public override System.IAsyncResult BeginRead(byte[] array, int offset, int count, System.AsyncCallback? asyncCallback, object? asyncState) { throw null; } + public override System.IAsyncResult BeginWrite(byte[] array, int offset, int count, System.AsyncCallback? asyncCallback, object? asyncState) { throw null; } + public override void CopyTo(System.IO.Stream destination, int bufferSize) { } + public override System.Threading.Tasks.Task CopyToAsync(System.IO.Stream destination, int bufferSize, System.Threading.CancellationToken cancellationToken) { throw null; } + protected override void Dispose(bool disposing) { } + public override System.Threading.Tasks.ValueTask DisposeAsync() { throw null; } + public override int EndRead(System.IAsyncResult asyncResult) { throw null; } + public override void EndWrite(System.IAsyncResult asyncResult) { } + public override void Flush() { } + public override System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; } + public override int Read(byte[] array, int offset, int count) { throw null; } + public override int Read(System.Span buffer) { throw null; } + public override System.Threading.Tasks.Task ReadAsync(byte[] array, int offset, int count, System.Threading.CancellationToken cancellationToken) { throw null; } + public override System.Threading.Tasks.ValueTask ReadAsync(System.Memory buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public override int ReadByte() { throw null; } + public override long Seek(long offset, System.IO.SeekOrigin origin) { throw null; } + public override void SetLength(long value) { } + public override void Write(byte[] array, int offset, int count) { } + public override void Write(System.ReadOnlySpan buffer) { } + public override void WriteByte(byte value) { } + public override System.Threading.Tasks.Task WriteAsync(byte[] array, int offset, int count, System.Threading.CancellationToken cancellationToken) { throw null; } + public override System.Threading.Tasks.ValueTask WriteAsync(System.ReadOnlyMemory buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + } } diff --git a/src/libraries/System.IO.Compression/src/System.IO.Compression.csproj b/src/libraries/System.IO.Compression/src/System.IO.Compression.csproj index e22bba9..1bf09a4 100644 --- a/src/libraries/System.IO.Compression/src/System.IO.Compression.csproj +++ b/src/libraries/System.IO.Compression/src/System.IO.Compression.csproj @@ -1,4 +1,4 @@ - + true $(NetCoreAppCurrent)-Windows_NT;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser @@ -33,6 +33,7 @@ + diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs index 437dd4b..4a34fe8 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs @@ -320,7 +320,7 @@ namespace System.IO.Compression private static void ThrowStreamClosedException() { - throw new ObjectDisposedException(null, SR.ObjectDisposed_StreamClosed); + throw new ObjectDisposedException(nameof(DeflateStream), SR.ObjectDisposed_StreamClosed); } private void EnsureDecompressionMode() diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/ZLibNative.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/ZLibNative.cs index c6d03a6..92e9e00 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/ZLibNative.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/ZLibNative.cs @@ -115,6 +115,14 @@ namespace System.IO.Compression // negative val causes deflate to produce raw deflate data (no zlib header). /// + ///

From the ZLib manual:

+ ///

ZLib's windowBits parameter is the base two logarithm of the window size (the size of the history buffer). + /// It should be in the range 8..15 for this version of the library. Larger values of this parameter result in better compression + /// at the expense of memory usage. The default value is 15 if deflateInit is used instead.

+ /// + public const int ZLib_DefaultWindowBits = 15; + + /// ///

Zlib's windowBits parameter is the base two logarithm of the window size (the size of the history buffer). /// For GZip header encoding, windowBits should be equal to a value between 8..15 (to specify Window Size) added to /// 16. The range of values for GZip encoding is therefore 24..31. diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/GZipStream.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/GZipStream.cs index e09c000..bbdf8b3 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/GZipStream.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/GZipStream.cs @@ -234,7 +234,7 @@ namespace System.IO.Compression private static void ThrowStreamClosedException() { - throw new ObjectDisposedException(null, SR.ObjectDisposed_StreamClosed); + throw new ObjectDisposedException(nameof(GZipStream), SR.ObjectDisposed_StreamClosed); } } } diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/ZLibStream.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/ZLibStream.cs new file mode 100644 index 0000000..73cb438 --- /dev/null +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/ZLibStream.cs @@ -0,0 +1,293 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace System.IO.Compression +{ + ///

Provides methods and properties used to compress and decompress streams by using the zlib data format specification. + public sealed class ZLibStream : Stream + { + /// The underlying deflate stream. + private DeflateStream _deflateStream; + + /// Initializes a new instance of the class by using the specified stream and compression mode. + /// The stream to which compressed data is written or from which decompressed data is read. + /// One of the enumeration values that indicates whether to compress or decompress the stream. + public ZLibStream(Stream stream, CompressionMode mode) : this(stream, mode, leaveOpen: false) + { + } + + /// Initializes a new instance of the class by using the specified stream, compression mode, and whether to leave the open. + /// The stream to which compressed data is written or from which decompressed data is read. + /// One of the enumeration values that indicates whether to compress or decompress the stream. + /// to leave the stream object open after disposing the object; otherwise, . + public ZLibStream(Stream stream, CompressionMode mode, bool leaveOpen) + { + _deflateStream = new DeflateStream(stream, mode, leaveOpen, ZLibNative.ZLib_DefaultWindowBits); + } + + /// Initializes a new instance of the class by using the specified stream and compression level. + /// The stream to which compressed data is written. + /// One of the enumeration values that indicates whether to emphasize speed or compression efficiency when compressing the stream. + public ZLibStream(Stream stream, CompressionLevel compressionLevel) : this(stream, compressionLevel, leaveOpen: false) + { + } + + /// Initializes a new instance of the class by using the specified stream, compression level, and whether to leave the open. + /// The stream to which compressed data is written. + /// One of the enumeration values that indicates whether to emphasize speed or compression efficiency when compressing the stream. + /// to leave the stream object open after disposing the object; otherwise, . + public ZLibStream(Stream stream, CompressionLevel compressionLevel, bool leaveOpen) + { + _deflateStream = new DeflateStream(stream, compressionLevel, leaveOpen, ZLibNative.ZLib_DefaultWindowBits); + } + + /// Gets a value indicating whether the stream supports reading. + public override bool CanRead => _deflateStream?.CanRead ?? false; + + /// Gets a value indicating whether the stream supports writing. + public override bool CanWrite => _deflateStream?.CanWrite ?? false; + + /// Gets a value indicating whether the stream supports seeking. + public override bool CanSeek => false; + + /// This property is not supported and always throws a . + public override long Length => throw new NotSupportedException(); + + /// This property is not supported and always throws a . + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + /// Flushes the internal buffers. + public override void Flush() + { + ThrowIfClosed(); + _deflateStream.Flush(); + } + + /// Asynchronously clears all buffers for this stream, causes any buffered data to be written to the underlying device, and monitors cancellation requests. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous flush operation. + public override Task FlushAsync(CancellationToken cancellationToken) + { + ThrowIfClosed(); + return _deflateStream.FlushAsync(cancellationToken); + } + + /// This method is not supported and always throws a . + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + /// This method is not supported and always throws a . + public override void SetLength(long value) => throw new NotSupportedException(); + + /// Reads a byte from the stream and advances the position within the stream by one byte, or returns -1 if at the end of the stream. + /// The unsigned byte cast to an , or -1 if at the end of the stream. + public override int ReadByte() + { + ThrowIfClosed(); + return _deflateStream.ReadByte(); + } + + /// Begins an asynchronous read operation. + /// The byte array to read the data into. + /// The byte offset in array at which to begin reading data from the stream. + /// The maximum number of bytes to read. + /// An optional asynchronous callback, to be called when the read operation is complete. + /// A user-provided object that distinguishes this particular asynchronous read request from other requests. + /// An object that represents the asynchronous read operation, which could still be pending. + public override IAsyncResult BeginRead(byte[] array, int offset, int count, AsyncCallback? asyncCallback, object? asyncState) + { + ThrowIfClosed(); + return _deflateStream.BeginRead(array, offset, count, asyncCallback, asyncState); + } + + /// Waits for the pending asynchronous read to complete. + /// The reference to the pending asynchronous request to finish. + /// The number of bytes that were read into the byte array. + public override int EndRead(IAsyncResult asyncResult) => + _deflateStream.EndRead(asyncResult); + + /// Reads a number of decompressed bytes into the specified byte array. + /// The byte array to read the data into. + /// The byte offset in array at which to begin reading data from the stream. + /// The maximum number of bytes to read. + /// The number of bytes that were read into the byte array. + public override int Read(byte[] array, int offset, int count) + { + ThrowIfClosed(); + return _deflateStream.Read(array, offset, count); + } + + /// Reads a number of decompressed bytes into the specified byte span. + /// The span to read the data into. + /// The number of bytes that were read into the byte span. + public override int Read(Span buffer) + { + ThrowIfClosed(); + return _deflateStream.ReadCore(buffer); + } + + /// Asynchronously reads a sequence of bytes from the current stream, advances the position within the stream by the number of bytes read, and monitors cancellation requests. + /// The byte array to read the data into. + /// The byte offset in array at which to begin reading data from the stream. + /// The maximum number of bytes to read. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous completion of the operation. + public override Task ReadAsync(byte[] array, int offset, int count, CancellationToken cancellationToken) + { + ThrowIfClosed(); + return _deflateStream.ReadAsync(array, offset, count, cancellationToken); + } + + /// Asynchronously reads a sequence of bytes from the current stream, advances the position within the stream by the number of bytes read, and monitors cancellation requests. + /// The byte span to read the data into. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous completion of the operation. + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + ThrowIfClosed(); + return _deflateStream.ReadAsyncMemory(buffer, cancellationToken); + } + + /// Begins an asynchronous write operation. + /// The buffer to write data from. + /// The byte offset in buffer to begin writing from. + /// The maximum number of bytes to write. + /// An optional asynchronous callback, to be called when the write operation is complete. + /// A user-provided object that distinguishes this particular asynchronous write request from other requests. + /// An object that represents the asynchronous write operation, which could still be pending. + public override IAsyncResult BeginWrite(byte[] array, int offset, int count, AsyncCallback? asyncCallback, object? asyncState) + { + ThrowIfClosed(); + return _deflateStream.BeginWrite(array, offset, count, asyncCallback, asyncState); + } + + /// Ends an asynchronous write operation. + /// The reference to the pending asynchronous request to finish. + public override void EndWrite(IAsyncResult asyncResult) => + _deflateStream.EndWrite(asyncResult); + + /// Writes compressed bytes to the underlying stream from the specified byte array. + /// The buffer to write data from. + /// The byte offset in buffer to begin writing from. + /// The maximum number of bytes to write. + public override void Write(byte[] array, int offset, int count) + { + ThrowIfClosed(); + _deflateStream.Write(array, offset, count); + } + + /// Writes compressed bytes to the underlying stream from the specified byte span. + /// The buffer to write data from. + public override void Write(ReadOnlySpan buffer) + { + ThrowIfClosed(); + _deflateStream.WriteCore(buffer); + } + + /// Asynchronously writes a sequence of bytes to the current stream, advances the current position within this stream by the number of bytes written, and monitors cancellation requests. + /// The buffer to write data from. + /// The byte offset in buffer to begin writing from. + /// The maximum number of bytes to write. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous completion of the operation. + public override Task WriteAsync(byte[] array, int offset, int count, CancellationToken cancellationToken) + { + ThrowIfClosed(); + return _deflateStream.WriteAsync(array, offset, count, cancellationToken); + } + + /// Asynchronously writes a sequence of bytes to the current stream, advances the current position within this stream by the number of bytes written, and monitors cancellation requests. + /// The buffer to write data from. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous completion of the operation. + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + ThrowIfClosed(); + return _deflateStream.WriteAsyncMemory(buffer, cancellationToken); + } + + /// Writes a byte to the current position in the stream and advances the position within the stream by one byte. + /// The byte to write to the stream. + public override void WriteByte(byte value) + { + ThrowIfClosed(); + _deflateStream.WriteByte(value); + } + + /// Reads the bytes from the current stream and writes them to another stream, using the specified buffer size. + /// The stream to which the contents of the current stream will be copied. + /// The size of the buffer. This value must be greater than zero. + public override void CopyTo(Stream destination, int bufferSize) + { + ThrowIfClosed(); + _deflateStream.CopyTo(destination, bufferSize); + } + + /// Asynchronously reads the bytes from the current stream and writes them to another stream, using a specified buffer size and cancellation token. + /// The stream to which the contents of the current stream will be copied. + /// The size, in bytes, of the buffer. This value must be greater than zero. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous copy operation. + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { + ThrowIfClosed(); + return _deflateStream.CopyToAsync(destination, bufferSize, cancellationToken); + } + + /// Releases all resources used by the . + /// Whether this method is being called from Dispose. + protected override void Dispose(bool disposing) + { + try + { + if (disposing) + { + _deflateStream?.Dispose(); + } + _deflateStream = null!; + } + finally + { + base.Dispose(disposing); + } + } + + /// Asynchronously releases all resources used by the . + /// A task that represents the completion of the disposal operation. + public override ValueTask DisposeAsync() + { + DeflateStream? ds = _deflateStream; + if (ds is not null) + { + _deflateStream = null!; + return ds.DisposeAsync(); + } + + return default; + } + + /// Gets a reference to the underlying stream. + public Stream BaseStream => _deflateStream?.BaseStream!; + + /// Throws an if the stream is closed. + private void ThrowIfClosed() + { + if (_deflateStream is null) + { + ThrowClosedException(); + } + } + + /// Throws an . + [DoesNotReturn] + private static void ThrowClosedException() => + throw new ObjectDisposedException(nameof(ZLibStream), SR.ObjectDisposed_StreamClosed); + } +} diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs index 32e56cd..493bb6b 100644 --- a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -16,34 +18,27 @@ namespace System.IO.Compression public override Stream BaseStream(Stream stream) => ((DeflateStream)stream).BaseStream; protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("DeflateTestData", Path.GetFileName(uncompressedPath)); - /// - /// Test to pass gzipstream data to a deflatestream - /// - [Theory] - [MemberData(nameof(UncompressedTestFiles))] - public async Task DecompressFailsWithRealGzStream(string uncompressedPath) + public static IEnumerable DecompressFailsWithWrapperStream_MemberData() { - string fileName = Path.Combine("GZipTestData", Path.GetFileName(uncompressedPath) + ".gz"); - var baseStream = await LocalMemoryStream.readAppFileAsync(fileName); - var zip = CreateStream(baseStream, CompressionMode.Decompress); - int _bufferSize = 2048; - var bytes = new byte[_bufferSize]; - Assert.Throws(() => { zip.Read(bytes, 0, _bufferSize); }); - zip.Dispose(); + foreach (object[] testFile in UncompressedTestFiles()) + { + yield return new object[] { testFile[0], "GZipTestData", ".gz" }; + yield return new object[] { testFile[0], "ZLibTestData", ".z" }; + } } - [Fact] - public void Precancellation() + /// Test to pass GZipStream data and ZLibStream data to a DeflateStream + [Theory] + [MemberData(nameof(DecompressFailsWithWrapperStream_MemberData))] + public async Task DecompressFailsWithWrapperStream(string uncompressedPath, string newDirectory, string newSuffix) { - var ms = new MemoryStream(); - using (Stream compressor = new DeflateStream(ms, CompressionMode.Compress, leaveOpen: true)) + string fileName = Path.Combine(newDirectory, Path.GetFileName(uncompressedPath) + newSuffix); + using (LocalMemoryStream baseStream = await LocalMemoryStream.readAppFileAsync(fileName)) + using (Stream cs = CreateStream(baseStream, CompressionMode.Decompress)) { - Assert.True(compressor.WriteAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled); - Assert.True(compressor.FlushAsync(new CancellationToken(true)).IsCanceled); - } - using (Stream decompressor = CreateStream(ms, CompressionMode.Decompress, leaveOpen: true)) - { - Assert.True(decompressor.ReadAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled); + int _bufferSize = 2048; + var bytes = new byte[_bufferSize]; + Assert.Throws(() => { cs.Read(bytes, 0, _bufferSize); }); } } @@ -76,51 +71,9 @@ namespace System.IO.Compression } } - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task DisposeAsync_Flushes(bool derived, bool leaveOpen) - { - var ms = new MemoryStream(); - var ds = derived ? - new DerivedDeflateStream(ms, CompressionMode.Compress, leaveOpen) : - new DeflateStream(ms, CompressionMode.Compress, leaveOpen); - ds.WriteByte(1); - Assert.Equal(0, ms.Position); - await ds.DisposeAsync(); - Assert.InRange(ms.ToArray().Length, 1, int.MaxValue); - if (leaveOpen) - { - Assert.InRange(ms.Position, 1, int.MaxValue); - } - else - { - Assert.Throws(() => ms.Position); - } - } - - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task DisposeAsync_MultipleCallsAllowed(bool derived, bool leaveOpen) - { - using (var ds = derived ? - new DerivedDeflateStream(new MemoryStream(), CompressionMode.Compress, leaveOpen) : - new DeflateStream(new MemoryStream(), CompressionMode.Compress, leaveOpen)) - { - await ds.DisposeAsync(); - await ds.DisposeAsync(); - } - } - private sealed class DerivedDeflateStream : DeflateStream { public bool ReadArrayInvoked = false, WriteArrayInvoked = false; - internal DerivedDeflateStream(Stream stream, CompressionMode mode) : base(stream, mode) { } internal DerivedDeflateStream(Stream stream, CompressionMode mode, bool leaveOpen) : base(stream, mode, leaveOpen) { } public override int Read(byte[] array, int offset, int count) diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs index 1963486..b3a45d0 100644 --- a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Gzip.cs @@ -18,21 +18,6 @@ namespace System.IO.Compression protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("GZipTestData", Path.GetFileName(uncompressedPath) + ".gz"); [Fact] - public void Precancellation() - { - var ms = new MemoryStream(); - using (Stream compressor = new GZipStream(ms, CompressionMode.Compress, leaveOpen: true)) - { - Assert.True(compressor.WriteAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled); - Assert.True(compressor.FlushAsync(new CancellationToken(true)).IsCanceled); - } - using (Stream decompressor = CreateStream(ms, CompressionMode.Decompress, leaveOpen: true)) - { - Assert.True(decompressor.ReadAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled); - } - } - - [Fact] public void ConcatenatedGzipStreams() { using (MemoryStream concatStream = new MemoryStream()) @@ -63,7 +48,8 @@ namespace System.IO.Compression /// that bypasses buffering. /// private class DerivedMemoryStream : MemoryStream - { } + { + } [Fact] public async Task ConcatenatedEmptyGzipStreams() @@ -295,52 +281,6 @@ namespace System.IO.Compression } } - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task DisposeAsync_Flushes(bool derived, bool leaveOpen) - { - var ms = new MemoryStream(); - var gs = derived ? - new DerivedGZipStream(ms, CompressionMode.Compress, leaveOpen) : - new GZipStream(ms, CompressionMode.Compress, leaveOpen); - gs.WriteByte(1); - await gs.FlushAsync(); - - long pos = ms.Position; - gs.WriteByte(1); - Assert.Equal(pos, ms.Position); - - await gs.DisposeAsync(); - Assert.InRange(ms.ToArray().Length, pos + 1, int.MaxValue); - if (leaveOpen) - { - Assert.InRange(ms.Position, pos + 1, int.MaxValue); - } - else - { - Assert.Throws(() => ms.Position); - } - } - - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public async Task DisposeAsync_MultipleCallsAllowed(bool derived, bool leaveOpen) - { - using (var gs = derived ? - new DerivedGZipStream(new MemoryStream(), CompressionMode.Compress, leaveOpen) : - new GZipStream(new MemoryStream(), CompressionMode.Compress, leaveOpen)) - { - await gs.DisposeAsync(); - await gs.DisposeAsync(); - } - } - private sealed class DerivedGZipStream : GZipStream { public bool ReadArrayInvoked = false, WriteArrayInvoked = false; diff --git a/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs new file mode 100644 index 0000000..5f43177 --- /dev/null +++ b/src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.ZLib.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace System.IO.Compression +{ + public class ZLibStreamUnitTests : CompressionStreamUnitTestBase + { + public override Stream CreateStream(Stream stream, CompressionMode mode) => new ZLibStream(stream, mode); + public override Stream CreateStream(Stream stream, CompressionMode mode, bool leaveOpen) => new ZLibStream(stream, mode, leaveOpen); + public override Stream CreateStream(Stream stream, CompressionLevel level) => new ZLibStream(stream, level); + public override Stream CreateStream(Stream stream, CompressionLevel level, bool leaveOpen) => new ZLibStream(stream, level, leaveOpen); + public override Stream BaseStream(Stream stream) => ((ZLibStream)stream).BaseStream; + protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("ZLibTestData", Path.GetFileName(uncompressedPath) + ".z"); + } +} diff --git a/src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj b/src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj index f126aec..80fc263 100644 --- a/src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj +++ b/src/libraries/System.IO.Compression/tests/System.IO.Compression.Tests.csproj @@ -3,6 +3,7 @@ $(NetCoreAppCurrent)-Windows_NT;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser + @@ -14,26 +15,16 @@ - - - - - - - - - - + + + + + + + + + + diff --git a/src/libraries/System.Net.Http.WinHttpHandler/tests/UnitTests/TestServer.cs b/src/libraries/System.Net.Http.WinHttpHandler/tests/UnitTests/TestServer.cs index ca8b263..64b3a50 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/tests/UnitTests/TestServer.cs +++ b/src/libraries/System.Net.Http.WinHttpHandler/tests/UnitTests/TestServer.cs @@ -158,18 +158,12 @@ namespace System.Net.Http.WinHttpHandlerUnitTests { using (var memoryStream = new MemoryStream()) { - Stream compressedStream = null; - if (useGZip) + using (Stream compressedStream = useGZip ? + new GZipStream(memoryStream, CompressionMode.Compress) : + new DeflateStream(memoryStream, CompressionMode.Compress)) { - compressedStream = new GZipStream(memoryStream, CompressionMode.Compress); + compressedStream.Write(bytes, 0, bytes.Length); } - else - { - compressedStream = new DeflateStream(memoryStream, CompressionMode.Compress); - } - - compressedStream.Write(bytes, 0, bytes.Length); - compressedStream.Dispose(); return memoryStream.ToArray(); } diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/DecompressionHandler.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/DecompressionHandler.cs index 5bac88a..08ecfde 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/DecompressionHandler.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/DecompressionHandler.cs @@ -211,7 +211,12 @@ namespace System.Net.Http { } protected override Stream GetDecompressedStream(Stream originalStream) => - new DeflateStream(originalStream, CompressionMode.Decompress); + // As described in RFC 2616, the deflate content-coding is actually + // the "zlib" format (RFC 1950) in combination with the "deflate" + // compression algrithm (RFC 1951). So while potentially + // counterintuitive based on naming, this needs to use ZLibStream + // rather than DeflateStream. + new ZLibStream(originalStream, CompressionMode.Decompress); } private sealed class BrotliDecompressedContent : DecompressedContent -- 2.7.4