// The .NET Foundation licenses this file to you under the MIT license.
using System.Collections.Generic;
+using System.IO.Compression.Tests;
+using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
+using Xunit.Sdk;
namespace System.IO.Compression
{
{
private const int TaskTimeout = 30 * 1000; // Generous timeout for official test runs
+ public enum TestScenario
+ {
+ ReadByte,
+ ReadByteAsync,
+ Read,
+ ReadAsync,
+ Copy,
+ CopyAsync,
+ }
+
[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))]
public virtual void FlushAsync_DuringWriteAsync()
{
Assert.True(fastestLength >= optimalLength);
Assert.True(optimalLength >= smallestLength);
}
+
+ [Theory]
+ [InlineData(TestScenario.ReadAsync)]
+ [InlineData(TestScenario.Read)]
+ [InlineData(TestScenario.Copy)]
+ [InlineData(TestScenario.CopyAsync)]
+ [InlineData(TestScenario.ReadByte)]
+ [InlineData(TestScenario.ReadByteAsync)]
+ public async Task StreamTruncation_IsDetected(TestScenario scenario)
+ {
+ var buffer = new byte[16];
+ byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray();
+ byte[] compressedData;
+ using (var compressed = new MemoryStream())
+ using (Stream compressor = CreateStream(compressed, CompressionMode.Compress))
+ {
+ foreach (byte b in source)
+ {
+ compressor.WriteByte(b);
+ }
+
+ compressor.Dispose();
+ compressedData = compressed.ToArray();
+ }
+
+ for (var i = 1; i <= compressedData.Length; i += 1)
+ {
+ bool expectException = i < compressedData.Length;
+ using (var compressedStream = new MemoryStream(compressedData.Take(i).ToArray()))
+ {
+ using (Stream decompressor = CreateStream(compressedStream, CompressionMode.Decompress))
+ {
+ var decompressedStream = new MemoryStream();
+
+ try
+ {
+ switch (scenario)
+ {
+ case TestScenario.Copy:
+ decompressor.CopyTo(decompressedStream);
+ break;
+
+ case TestScenario.CopyAsync:
+ await decompressor.CopyToAsync(decompressedStream);
+ break;
+
+ case TestScenario.Read:
+ while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0) { };
+ break;
+
+ case TestScenario.ReadAsync:
+ while (await ZipFileTestBase.ReadAllBytesAsync(decompressor, buffer, 0, buffer.Length) != 0) { };
+ break;
+
+ case TestScenario.ReadByte:
+ while (decompressor.ReadByte() != -1) { }
+ break;
+
+ case TestScenario.ReadByteAsync:
+ while (await decompressor.ReadByteAsync() != -1) { }
+ break;
+ }
+ }
+ catch (InvalidDataException e)
+ {
+ if (expectException)
+ continue;
+
+ throw new XunitException($"An unexpected error occured while decompressing data:{e}");
+ }
+
+ if (expectException)
+ {
+ throw new XunitException($"Truncated stream was decompressed successfully but exception was expected: length={i}/{compressedData.Length}");
+ }
+ }
+ }
+ }
+ }
}
internal sealed class BadWrappedStream : MemoryStream
}
}
+ public static async Task<int> ReadAllBytesAsync(Stream stream, byte[] buffer, int offset, int count)
+ {
+ int bytesRead;
+ int totalRead = 0;
+ while ((bytesRead = await stream.ReadAsync(buffer, offset + totalRead, count - totalRead)) != 0)
+ {
+ totalRead += bytesRead;
+ }
+ return totalRead;
+ }
+
public static int ReadAllBytes(Stream stream, byte[] buffer, int offset, int count)
{
int bytesRead;
StreamsEqual(ast, bst, -1);
}
+ public static async Task StreamsEqualAsync(Stream ast, Stream bst)
+ {
+ await StreamsEqualAsync(ast, bst, -1);
+ }
+
public static void StreamsEqual(Stream ast, Stream bst, int blocksToRead)
{
if (ast.CanSeek)
if (blocksToRead != -1 && blocksRead >= blocksToRead)
break;
- ac = ReadAllBytes(ast, ad, 0, 4096);
- bc = ReadAllBytes(bst, bd, 0, 4096);
+ ac = ReadAllBytes(ast, ad, 0, bufSize);
+ bc = ReadAllBytes(bst, bd, 0, bufSize);
+
+ if (ac != bc)
+ {
+ bd = NormalizeLineEndings(bd);
+ }
+
+ Assert.True(ArraysEqual<byte>(ad, bd, ac), "Stream contents not equal: " + ast.ToString() + ", " + bst.ToString());
+
+ blocksRead++;
+ } while (ac == bufSize);
+ }
+
+ public static async Task StreamsEqualAsync(Stream ast, Stream bst, int blocksToRead)
+ {
+ if (ast.CanSeek)
+ ast.Seek(0, SeekOrigin.Begin);
+ if (bst.CanSeek)
+ bst.Seek(0, SeekOrigin.Begin);
+
+ const int bufSize = 4096;
+ byte[] ad = new byte[bufSize];
+ byte[] bd = new byte[bufSize];
+
+ int ac = 0;
+ int bc = 0;
+
+ int blocksRead = 0;
+
+ //assume read doesn't do weird things
+ do
+ {
+ if (blocksToRead != -1 && blocksRead >= blocksToRead)
+ break;
+
+ ac = await ReadAllBytesAsync(ast, ad, 0, bufSize);
+ bc = await ReadAllBytesAsync(bst, bd, 0, bufSize);
if (ac != bc)
{
Assert.True(ArraysEqual<byte>(ad, bd, ac), "Stream contents not equal: " + ast.ToString() + ", " + bst.ToString());
blocksRead++;
- } while (ac == 4096);
+ } while (ac == bufSize);
}
public static async Task IsZipSameAsDirAsync(string archiveFile, string directory, ZipArchiveMode mode)
<data name="BrotliStream_Decompress_InvalidStream" xml:space="preserve">
<value>BrotliStream.BaseStream returned more bytes than requested in Read.</value>
</data>
+ <data name="BrotliStream_Decompress_TruncatedData" xml:space="preserve">
+ <value>Decoder ran into truncated data.</value>
+ </data>
<data name="IOCompressionBrotli_PlatformNotSupported" xml:space="preserve">
<value>System.IO.Compression.Brotli is not supported on this platform.</value>
</data>
private BrotliDecoder _decoder;
private int _bufferOffset;
private int _bufferCount;
+ private bool _nonEmptyInput;
/// <summary>Reads a number of decompressed bytes into the specified byte array.</summary>
/// <param name="buffer">The array used to store decompressed bytes.</param>
int bytesRead = _stream.Read(_buffer, _bufferCount, _buffer.Length - _bufferCount);
if (bytesRead <= 0)
{
+ if (_nonEmptyInput && !buffer.IsEmpty)
+ ThrowTruncatedInvalidData();
break;
}
+ _nonEmptyInput = true;
_bufferCount += bytesRead;
if (_bufferCount > _buffer.Length)
int bytesRead = await _stream.ReadAsync(_buffer.AsMemory(_bufferCount), cancellationToken).ConfigureAwait(false);
if (bytesRead <= 0)
{
+ if (_nonEmptyInput && !buffer.IsEmpty)
+ ThrowTruncatedInvalidData();
break;
}
+ _nonEmptyInput = true;
_bufferCount += bytesRead;
if (_bufferCount > _buffer.Length)
// The stream is either malicious or poorly implemented and returned a number of
// bytes larger than the buffer supplied to it.
throw new InvalidDataException(SR.BrotliStream_Decompress_InvalidStream);
+
+ private static void ThrowTruncatedInvalidData() =>
+ throw new InvalidDataException(SR.BrotliStream_Decompress_TruncatedData);
}
}
-<Project Sdk="Microsoft.NET.Sdk">
+
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>$(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser</TargetFrameworks>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
Link="Common\System\IO\Compression\CompressionStreamTestBase.cs" />
<Compile Include="$(CommonTestPath)System\IO\Compression\CompressionStreamUnitTestBase.cs"
Link="Common\System\IO\Compression\CompressionStreamUnitTestBase.cs" />
+ <Compile Include="$(CommonTestPath)System\IO\Compression\CRC.cs"
+ Link="Common\System\IO\Compression\CRC.cs" />
+ <Compile Include="$(CommonTestPath)System\IO\Compression\FileData.cs"
+ Link="Common\System\IO\Compression\FileData.cs" />
<Compile Include="$(CommonTestPath)System\IO\Compression\LocalMemoryStream.cs"
Link="Common\System\IO\Compression\LocalMemoryStream.cs" />
<Compile Include="$(CommonTestPath)System\IO\Compression\StreamHelpers.cs"
Link="Common\System\IO\Compression\StreamHelpers.cs" />
+ <Compile Include="$(CommonTestPath)System\IO\Compression\ZipTestHelper.cs"
+ Link="Common\System\IO\Compression\ZipTestHelper.cs" />
<Compile Include="$(CommonTestPath)System\IO\TempFile.cs"
Link="Common\System\IO\TempFile.cs" />
<Compile Include="$(CommonPath)System\Threading\Tasks\TaskToApm.cs"
}
[Fact]
+ public async Task CreateFromDirectory_IncludeBaseDirectoryAsync()
+ {
+ string folderName = zfolder("normal");
+ string withBaseDir = GetTestFilePath();
+ ZipFile.CreateFromDirectory(folderName, withBaseDir, CompressionLevel.Optimal, true);
+
+ IEnumerable<string> expected = Directory.EnumerateFiles(zfolder("normal"), "*", SearchOption.AllDirectories);
+ using (ZipArchive actual_withbasedir = ZipFile.Open(withBaseDir, ZipArchiveMode.Read))
+ {
+ foreach (ZipArchiveEntry actualEntry in actual_withbasedir.Entries)
+ {
+ string expectedFile = expected.Single(i => Path.GetFileName(i).Equals(actualEntry.Name));
+ Assert.StartsWith("normal", actualEntry.FullName);
+ Assert.Equal(new FileInfo(expectedFile).Length, actualEntry.Length);
+ using (Stream expectedStream = File.OpenRead(expectedFile))
+ using (Stream actualStream = actualEntry.Open())
+ {
+ await StreamsEqualAsync(expectedStream, actualStream);
+ }
+ }
+ }
+ }
+
+ [Fact]
public void CreateFromDirectoryUnicode()
{
string folderName = zfolder("unicode");
<data name="SplitSpanned" xml:space="preserve">
<value>Split or spanned archives are not supported.</value>
</data>
+ <data name="TruncatedData" xml:space="preserve">
+ <value>Found truncated data while decoding.</value>
+ </data>
<data name="UnexpectedEndOfStream" xml:space="preserve">
<value>Zip file corrupt: unexpected end of stream reached.</value>
</data>
int n = _stream.Read(_buffer, 0, _buffer.Length);
if (n <= 0)
{
+ // - Inflater didn't return any data although a non-empty output buffer was passed by the caller.
+ // - More input is needed but there is no more input available.
+ // - Inflation is not finished yet.
+ // - Provided input wasn't completely empty
+ // In such case, we are dealing with a truncated input stream.
+ if (!buffer.IsEmpty && !_inflater.Finished() && _inflater.NonEmptyInput())
+ {
+ ThrowTruncatedInvalidData();
+ }
break;
}
else if (n > _buffer.Length)
// bytes < 0 || > than the buffer supplied to it.
throw new InvalidDataException(SR.GenericInvalidData);
+ private static void ThrowTruncatedInvalidData() =>
+ throw new InvalidDataException(SR.TruncatedData);
+
public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? asyncCallback, object? asyncState) =>
TaskToApm.Begin(ReadAsync(buffer, offset, count, CancellationToken.None), asyncCallback, asyncState);
int n = await _stream.ReadAsync(new Memory<byte>(_buffer, 0, _buffer.Length), cancellationToken).ConfigureAwait(false);
if (n <= 0)
{
+ // - Inflater didn't return any data although a non-empty output buffer was passed by the caller.
+ // - More input is needed but there is no more input available.
+ // - Inflation is not finished yet.
+ // - Provided input wasn't completely empty
+ // In such case, we are dealing with a truncated input stream.
+ if (!_inflater.Finished() && _inflater.NonEmptyInput() && !buffer.IsEmpty)
+ {
+ ThrowTruncatedInvalidData();
+ }
break;
}
else if (n > _buffer.Length)
// decompress into at least one byte of output, but it's a reasonable approximation for the 99% case. If it's
// wrong, it just means that a caller using zero-byte reads as a way to delay getting a buffer to use for a
// subsequent call may end up getting one earlier than otherwise preferred.
+ Debug.Assert(bytesRead == 0);
break;
}
}
// Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream
await _deflateStream._stream.CopyToAsync(this, _arrayPoolBuffer.Length, _cancellationToken).ConfigureAwait(false);
+ if (!_deflateStream._inflater.Finished())
+ {
+ ThrowTruncatedInvalidData();
+ }
}
finally
{
// Now, use the source stream's CopyToAsync to push directly to our inflater via this helper stream
_deflateStream._stream.CopyTo(this, _arrayPoolBuffer.Length);
+ if (!_deflateStream._inflater.Finished())
+ {
+ ThrowTruncatedInvalidData();
+ }
}
finally
{
private const int MinWindowBits = -15; // WindowBits must be between -8..-15 to ignore the header, 8..15 for
private const int MaxWindowBits = 47; // zlib headers, 24..31 for GZip headers, or 40..47 for either Zlib or GZip
+ private bool _nonEmptyInput; // Whether there is any non empty input
private bool _finished; // Whether the end of the stream has been reached
private bool _isDisposed; // Prevents multiple disposals
private readonly int _windowBits; // The WindowBits parameter passed to Inflater construction
{
Debug.Assert(windowBits >= MinWindowBits && windowBits <= MaxWindowBits);
_finished = false;
+ _nonEmptyInput = false;
_isDisposed = false;
_windowBits = windowBits;
InflateInit(windowBits);
public bool NeedsInput() => _zlibStream.AvailIn == 0;
+ public bool NonEmptyInput() => _nonEmptyInput;
+
public void SetInput(byte[] inputBuffer, int startIndex, int count)
{
Debug.Assert(NeedsInput(), "We have something left in previous input!");
_zlibStream.NextIn = (IntPtr)_inputBufferHandle.Pointer;
_zlibStream.AvailIn = (uint)inputBuffer.Length;
_finished = false;
+ _nonEmptyInput = true;
}
}
// The .NET Foundation licenses this file to you under the MIT license.
using System.Buffers;
+using System.Linq;
using System.Threading;
using System.Threading.Tasks;
+using System.IO.Compression.Tests;
using Xunit;
namespace System.IO.Compression
await TestConcatenatedGzipStreams(streamCount, scenario, bufferSize, bytesPerStream);
}
- public enum TestScenario
- {
- ReadByte,
- Read,
- ReadAsync,
- Copy,
- CopyAsync
- }
-
private async Task TestConcatenatedGzipStreams(int streamCount, TestScenario scenario, int bufferSize, int bytesPerStream = 1)
{
bool isCopy = scenario == TestScenario.Copy || scenario == TestScenario.CopyAsync;
}
}
+ [Fact]
+ public void StreamCorruption_IsDetected()
+ {
+ byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray();
+ var buffer = new byte[64];
+ byte[] compressedData;
+ using (var compressed = new MemoryStream())
+ using (Stream compressor = CreateStream(compressed, CompressionMode.Compress))
+ {
+ foreach (byte b in source)
+ {
+ compressor.WriteByte(b);
+ }
+
+ compressor.Dispose();
+ compressedData = compressed.ToArray();
+ }
+
+ // the last 7 bytes of the 10-byte gzip header can be changed with no decompression error
+ // this is by design, so we skip them for the test
+ int[] byteToSkip = { 3, 4, 5, 6, 7, 8, 9 };
+
+ for (int byteToCorrupt = 0; byteToCorrupt < compressedData.Length; byteToCorrupt++)
+ {
+ if (byteToSkip.Contains(byteToCorrupt))
+ continue;
+
+ // corrupt the data
+ compressedData[byteToCorrupt]++;
+
+ using (var decompressedStream = new MemoryStream(compressedData))
+ {
+ using (Stream decompressor = CreateStream(decompressedStream, CompressionMode.Decompress))
+ {
+ Assert.Throws<InvalidDataException>(() =>
+ {
+ while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0);
+ });
+ }
+ }
+
+ // restore the data
+ compressedData[byteToCorrupt]--;
+ }
+ }
+
private sealed class DerivedGZipStream : GZipStream
{
public bool ReadArrayInvoked = false, WriteArrayInvoked = false;
// The .NET Foundation licenses this file to you under the MIT license.
using System.Buffers;
+using System.IO.Compression.Tests;
+using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
public override Stream CreateStream(Stream stream, CompressionLevel level, bool leaveOpen) => new ZLibStream(stream, level, leaveOpen);
public override Stream BaseStream(Stream stream) => ((ZLibStream)stream).BaseStream;
protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("ZLibTestData", Path.GetFileName(uncompressedPath) + ".z");
+
+ [Fact]
+ public void StreamCorruption_IsDetected()
+ {
+ byte[] source = Enumerable.Range(0, 64).Select(i => (byte)i).ToArray();
+ var buffer = new byte[64];
+ byte[] compressedData;
+ using (var compressed = new MemoryStream())
+ using (Stream compressor = CreateStream(compressed, CompressionMode.Compress))
+ {
+ foreach (byte b in source)
+ {
+ compressor.WriteByte(b);
+ }
+
+ compressor.Dispose();
+ compressedData = compressed.ToArray();
+ }
+
+ for (int byteToCorrupt = 0; byteToCorrupt < compressedData.Length; byteToCorrupt++)
+ {
+ // corrupt the data
+ compressedData[byteToCorrupt]++;
+
+ using (var decompressedStream = new MemoryStream(compressedData))
+ {
+ using (Stream decompressor = CreateStream(decompressedStream, CompressionMode.Decompress))
+ {
+ Assert.Throws<InvalidDataException>(() =>
+ {
+ while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0);
+ });
+ }
+ }
+
+ // restore the data
+ compressedData[byteToCorrupt]--;
+ }
+ }
}
}
projects / platform / upstream / dotnet / runtime.git / commitdiff