Tar: Fix PAX regression when handling the size of really long unseekable data streams...
authorCarlos Sánchez López <1175054+carlossanlop@users.noreply.github.com>
Mon, 17 Jul 2023 22:19:08 +0000 (15:19 -0700)
committerGitHub <noreply@github.com>
Mon, 17 Jul 2023 22:19:08 +0000 (15:19 -0700)
* Fix regression introduced by https://github.com/dotnet/runtime/pull/84279 preventing PAX entries with really long data streams to get its size correctly stored in the extended attributes when the data stream is unseekable.

* Move tests for large files to a new manual tests project.

src/libraries/System.Formats.Tar/System.Formats.Tar.sln
src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs
src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarWriter.cs
src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs [new file with mode: 0644]
src/libraries/System.Formats.Tar/tests/Manual/ManualTestsAsync.cs [new file with mode: 0644]
src/libraries/System.Formats.Tar/tests/Manual/System.Formats.Tar.Manual.Tests.csproj [new file with mode: 0644]
src/libraries/System.Formats.Tar/tests/System.Formats.Tar.Tests.csproj
src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.LongFile.Tests.cs [deleted file]
src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.LongFile.Tests.cs [deleted file]

index fb37626..a5ff8f4 100644 (file)
@@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Formats.Tar", "src\S
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Formats.Tar.Tests", "tests\System.Formats.Tar.Tests.csproj", "{6FD1E284-7B50-4077-B73A-5B31CB0E3577}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Formats.Tar.Manual.Tests", "tests\Manual\System.Formats.Tar.Manual.Tests.csproj", "{D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ComInterfaceGenerator", "..\System.Runtime.InteropServices\gen\ComInterfaceGenerator\ComInterfaceGenerator.csproj", "{00477EA4-C3E5-48A9-8CA8-8CCF689E0DB4}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LibraryImportGenerator", "..\System.Runtime.InteropServices\gen\LibraryImportGenerator\LibraryImportGenerator.csproj", "{E89FEF3E-E0B9-41C4-A51C-9759AD1A3B69}"
@@ -67,6 +69,10 @@ Global
                {A00011A0-E609-4A49-B893-EBFC72C98707}.Debug|Any CPU.Build.0 = Debug|Any CPU
                {A00011A0-E609-4A49-B893-EBFC72C98707}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {A00011A0-E609-4A49-B893-EBFC72C98707}.Release|Any CPU.Build.0 = Release|Any CPU
+               {D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+               {D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+               {D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+               {D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}.Release|Any CPU.Build.0 = Release|Any CPU
        EndGlobalSection
        GlobalSection(SolutionProperties) = preSolution
                HideSolutionNode = FALSE
@@ -78,9 +84,12 @@ Global
                {E0B882C6-2082-45F2-806E-568461A61975} = {9BE8AFF4-D37B-49AF-AFD3-A15E514AC8AE}
                {A00011A0-E609-4A49-B893-EBFC72C98707} = {9BE8AFF4-D37B-49AF-AFD3-A15E514AC8AE}
                {9F751C2B-56DD-4604-A3F3-568627F8C006} = {55A8C7E4-925C-4F21-B68B-CEFC19137A4B}
+               {6FD1E284-7B50-4077-B73A-5B31CB0E3577} = {6CF0D830-3EE9-44B1-B548-EA8750AD7B3E}
                {00477EA4-C3E5-48A9-8CA8-8CCF689E0DB4} = {0345BAA8-92BC-4499-B550-21AC44910FD2}
                {E89FEF3E-E0B9-41C4-A51C-9759AD1A3B69} = {0345BAA8-92BC-4499-B550-21AC44910FD2}
                {50E6D5FD-0E06-4D07-966E-C28E5448A1D3} = {0345BAA8-92BC-4499-B550-21AC44910FD2}
+               {A00011A0-E609-4A49-B893-EBFC72C98707} = {9BE8AFF4-D37B-49AF-AFD3-A15E514AC8AE}
+               {D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D} = {6CF0D830-3EE9-44B1-B548-EA8750AD7B3E}
        EndGlobalSection
        GlobalSection(ExtensibilityGlobals) = postSolution
                SolutionGuid = {F9B8DA67-C83B-466D-907C-9541CDBDCFEF}
index 5d9d3a5..a14c124 100644 (file)
@@ -25,59 +25,48 @@ namespace System.Formats.Tar
         private const string GnuLongMetadataName = "././@LongLink";
         private const string ArgNameEntry = "entry";
 
-        internal void WriteAs(TarEntryFormat format, Stream archiveStream, Span<byte> buffer)
+        // Writes the entry in the order required to be able to obtain the seekable data stream size.
+        private void WriteWithSeekableDataStream(TarEntryFormat format, Stream archiveStream, Span<byte> buffer)
         {
-            Debug.Assert(format > TarEntryFormat.Unknown && format <= TarEntryFormat.Gnu);
-            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
+            Debug.Assert(format is > TarEntryFormat.Unknown and <= TarEntryFormat.Gnu);
+            Debug.Assert(_dataStream == null || _dataStream.CanSeek);
 
-            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
-            {
-                WriteWithUnseekableDataStreamAs(format, archiveStream, buffer);
-            }
-            else // Seek status of archive does not matter
-            {
-                long bytesToWrite = GetTotalDataBytesToWrite();
-                WriteFieldsToBuffer(format, bytesToWrite, buffer);
-                archiveStream.Write(buffer);
+            _size = GetTotalDataBytesToWrite();
+            WriteFieldsToBuffer(format, buffer);
+            archiveStream.Write(buffer);
 
-                if (_dataStream != null)
-                {
-                    WriteData(archiveStream, _dataStream, _size);
-                }
+            if (_dataStream != null)
+            {
+                WriteData(archiveStream, _dataStream);
             }
         }
 
-        internal async Task WriteAsAsync(TarEntryFormat format, Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
+        // Asynchronously writes the entry in the order required to be able to obtain the seekable data stream size.
+        private async Task WriteWithSeekableDataStreamAsync(TarEntryFormat format, Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
         {
-            Debug.Assert(format > TarEntryFormat.Unknown && format <= TarEntryFormat.Gnu);
-            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
+            Debug.Assert(format is > TarEntryFormat.Unknown and <= TarEntryFormat.Gnu);
+            Debug.Assert(_dataStream == null || _dataStream.CanSeek);
 
-            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
-            {
-                await WriteWithUnseekableDataStreamAsAsync(format, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
-            }
-            else // seek status of archive does not matter
-            {
-                long bytesToWrite = GetTotalDataBytesToWrite();
-                WriteFieldsToBuffer(format, bytesToWrite, buffer.Span);
-                await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
+            _size = GetTotalDataBytesToWrite();
+            WriteFieldsToBuffer(format, buffer.Span);
+            await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
 
-                if (_dataStream != null)
-                {
-                    await WriteDataAsync(archiveStream, _dataStream, _size, cancellationToken).ConfigureAwait(false);
-                }
+            if (_dataStream != null)
+            {
+                await WriteDataAsync(archiveStream, _dataStream, cancellationToken).ConfigureAwait(false);
             }
         }
 
-        private void WriteWithUnseekableDataStreamAs(TarEntryFormat format, Stream archiveStream, Span<byte> buffer)
+        // Writes into the specified destination stream the entry in the order required to be able to obtain the unseekable data stream size.
+        private void WriteWithUnseekableDataStream(TarEntryFormat format, Stream destinationStream, Span<byte> buffer, bool shouldAdvanceToEnd)
         {
             // When the data stream is unseekable, the order in which we write the entry data changes
-            Debug.Assert(archiveStream.CanSeek);
+            Debug.Assert(destinationStream.CanSeek);
             Debug.Assert(_dataStream != null);
             Debug.Assert(!_dataStream.CanSeek);
 
             // Store the start of the current entry's header, it'll be used later
-            long headerStartPosition = archiveStream.Position;
+            long headerStartPosition = destinationStream.Position;
 
             ushort dataLocation = format switch
             {
@@ -91,39 +80,42 @@ namespace System.Formats.Tar
             long dataStartPosition = headerStartPosition + dataLocation;
 
             // Move to the data start location and write the data
-            archiveStream.Seek(dataLocation, SeekOrigin.Current);
-            _dataStream.CopyTo(archiveStream); // The data gets copied from the current position
+            destinationStream.Seek(dataLocation, SeekOrigin.Current);
+            _dataStream.CopyTo(destinationStream); // The data gets copied from the current position
 
             // Get the new archive stream position, and the difference is the size of the data stream
-            long dataEndPosition = archiveStream.Position;
-            long actualLength = dataEndPosition - dataStartPosition;
+            long dataEndPosition = destinationStream.Position;
+            _size = dataEndPosition - dataStartPosition;
 
             // Write the padding now so that we can go back to writing the entry's header metadata
-            WriteEmptyPadding(archiveStream, actualLength);
+            WriteEmptyPadding(destinationStream);
 
             // Store the end of the current header, we will write the next one after this position
-            long endOfHeaderPosition = archiveStream.Position;
+            long endOfHeaderPosition = destinationStream.Position;
 
             // Go back to the start of the entry header to write the rest of the fields
-            archiveStream.Position = headerStartPosition;
+            destinationStream.Position = headerStartPosition;
 
-            WriteFieldsToBuffer(format, actualLength, buffer);
-            archiveStream.Write(buffer);
+            WriteFieldsToBuffer(format, buffer);
+            destinationStream.Write(buffer);
 
-            // Finally, move to the end of the header to continue with the next entry
-            archiveStream.Position = endOfHeaderPosition;
+            if (shouldAdvanceToEnd)
+            {
+                // Finally, move to the end of the header to continue with the next entry
+                destinationStream.Position = endOfHeaderPosition;
+            }
         }
 
-        // Asynchronously writes the entry in the order required to be able to obtain the unseekable data stream size.
-        private async Task WriteWithUnseekableDataStreamAsAsync(TarEntryFormat format, Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
+        // Asynchronously writes into the destination stream the entry in the order required to be able to obtain the unseekable data stream size.
+        private async Task WriteWithUnseekableDataStreamAsync(TarEntryFormat format, Stream destinationStream, Memory<byte> buffer, bool shouldAdvanceToEnd, CancellationToken cancellationToken)
         {
             // When the data stream is unseekable, the order in which we write the entry data changes
-            Debug.Assert(archiveStream.CanSeek);
+            Debug.Assert(destinationStream.CanSeek);
             Debug.Assert(_dataStream != null);
             Debug.Assert(!_dataStream.CanSeek);
 
             // Store the start of the current entry's header, it'll be used later
-            long headerStartPosition = archiveStream.Position;
+            long headerStartPosition = destinationStream.Position;
 
             ushort dataLocation = format switch
             {
@@ -137,33 +129,35 @@ namespace System.Formats.Tar
             long dataStartPosition = headerStartPosition + dataLocation;
 
             // Move to the data start location and write the data
-            archiveStream.Seek(dataLocation, SeekOrigin.Current);
-            await _dataStream.CopyToAsync(archiveStream, cancellationToken).ConfigureAwait(false); // The data gets copied from the current position
+            destinationStream.Seek(dataLocation, SeekOrigin.Current);
+            await _dataStream.CopyToAsync(destinationStream, cancellationToken).ConfigureAwait(false); // The data gets copied from the current position
 
             // Get the new archive stream position, and the difference is the size of the data stream
-            long dataEndPosition = archiveStream.Position;
-            long actualLength = dataEndPosition - dataStartPosition;
+            long dataEndPosition = destinationStream.Position;
+            _size = dataEndPosition - dataStartPosition;
 
             // Write the padding now so that we can go back to writing the entry's header metadata
-            await WriteEmptyPaddingAsync(archiveStream, actualLength, cancellationToken).ConfigureAwait(false);
+            await WriteEmptyPaddingAsync(destinationStream, cancellationToken).ConfigureAwait(false);
 
             // Store the end of the current header, we will write the next one after this position
-            long endOfHeaderPosition = archiveStream.Position;
+            long endOfHeaderPosition = destinationStream.Position;
 
             // Go back to the start of the entry header to write the rest of the fields
-            archiveStream.Position = headerStartPosition;
+            destinationStream.Position = headerStartPosition;
 
-            WriteFieldsToBuffer(format, actualLength, buffer.Span);
-            await archiveStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
+            WriteFieldsToBuffer(format, buffer.Span);
+            await destinationStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
 
-            // Finally, move to the end of the header to continue with the next entry
-            archiveStream.Position = endOfHeaderPosition;
+            if (shouldAdvanceToEnd)
+            {
+                // Finally, move to the end of the header to continue with the next entry
+                destinationStream.Position = endOfHeaderPosition;
+            }
         }
 
         // Writes the V7 header fields to the specified buffer, calculates and writes the checksum, then returns the final data length.
-        private void WriteV7FieldsToBuffer(long size, Span<byte> buffer)
+        private void WriteV7FieldsToBuffer(Span<byte> buffer)
         {
-            _size = size;
             TarEntryType actualEntryType = TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.V7, _typeFlag);
 
             int tmpChecksum = WriteName(buffer);
@@ -172,9 +166,8 @@ namespace System.Formats.Tar
         }
 
         // Writes the Ustar header fields to the specified buffer, calculates and writes the checksum, then returns the final data length.
-        private void WriteUstarFieldsToBuffer(long size, Span<byte> buffer)
+        private void WriteUstarFieldsToBuffer(Span<byte> buffer)
         {
-            _size = size;
             TarEntryType actualEntryType = TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Ustar, _typeFlag);
 
             int tmpChecksum = WriteUstarName(buffer);
@@ -211,51 +204,159 @@ namespace System.Formats.Tar
             Debug.Assert(globalExtendedAttributesEntryNumber >= 0);
         }
 
+        internal void WriteAsV7(Stream archiveStream, Span<byte> buffer)
+        {
+            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
+
+            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
+            {
+                WriteWithUnseekableDataStream(TarEntryFormat.V7, archiveStream, buffer, shouldAdvanceToEnd: true);
+            }
+            else // Seek status of archive does not matter
+            {
+                WriteWithSeekableDataStream(TarEntryFormat.V7, archiveStream, buffer);
+            }
+        }
+
+        internal Task WriteAsV7Async(Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
+        {
+            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
+
+            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
+            {
+                return WriteWithUnseekableDataStreamAsync(TarEntryFormat.V7, archiveStream, buffer, shouldAdvanceToEnd: true, cancellationToken);
+            }
+
+            // Else: Seek status of archive does not matter
+            return WriteWithSeekableDataStreamAsync(TarEntryFormat.V7, archiveStream, buffer, cancellationToken);
+        }
+
+        internal void WriteAsUstar(Stream archiveStream, Span<byte> buffer)
+        {
+            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
+
+            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
+            {
+                WriteWithUnseekableDataStream(TarEntryFormat.Ustar, archiveStream, buffer, shouldAdvanceToEnd: true);
+            }
+            else // Seek status of archive does not matter
+            {
+                WriteWithSeekableDataStream(TarEntryFormat.Ustar, archiveStream, buffer);
+            }
+        }
+
+        internal Task WriteAsUstarAsync(Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
+        {
+            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
+
+            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
+            {
+                return WriteWithUnseekableDataStreamAsync(TarEntryFormat.Ustar, archiveStream, buffer, shouldAdvanceToEnd: true, cancellationToken);
+            }
+
+            // Else: Seek status of archive does not matter
+            return WriteWithSeekableDataStreamAsync(TarEntryFormat.Ustar, archiveStream, buffer, cancellationToken);
+        }
+
         // Writes the current header as a PAX entry into the archive stream.
         // Makes sure to add the preceding extended attributes entry before the actual entry.
         internal void WriteAsPax(Stream archiveStream, Span<byte> buffer)
         {
+            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
             Debug.Assert(_typeFlag is not TarEntryType.GlobalExtendedAttributes);
 
-            // First, we write the preceding extended attributes header
+            // First, we create the preceding extended attributes header
             TarHeader extendedAttributesHeader = new(TarEntryFormat.Pax);
-            // Fill the current header's dict
-            CollectExtendedAttributesFromStandardFieldsIfNeeded();
-            // And pass the attributes to the preceding extended attributes header for writing
-            extendedAttributesHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1);
-            buffer.Clear(); // Reset it to reuse it
-            // Second, we write this header as a normal one
-            WriteAs(TarEntryFormat.Pax, archiveStream, buffer);
+
+            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
+            {
+                // Write the full entry header into a temporary stream, which will also collect the data length in the _size field
+                using MemoryStream tempStream = new();
+                // Don't advance the tempStream, instead, we will rewind it to the beginning for copying later
+                WriteWithUnseekableDataStream(TarEntryFormat.Pax, tempStream, buffer, shouldAdvanceToEnd: false);
+                tempStream.Position = 0;
+                buffer.Clear();
+
+                // If the data length is larger than it fits in the standard size field, it will get stored as an extended attribute
+                CollectExtendedAttributesFromStandardFieldsIfNeeded();
+
+                // Write the extended attributes entry into the archive first
+                extendedAttributesHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1);
+                buffer.Clear();
+
+                // And then write the stored entry into the archive
+                tempStream.CopyTo(archiveStream);
+            }
+            else // Seek status of archive does not matter
+            {
+                _size = GetTotalDataBytesToWrite();
+                // Fill the current header's dict
+                CollectExtendedAttributesFromStandardFieldsIfNeeded();
+                // And pass the attributes to the preceding extended attributes header for writing
+                extendedAttributesHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1);
+                buffer.Clear(); // Reset it to reuse it
+
+                // Second, we write this header as a normal one
+                WriteWithSeekableDataStream(TarEntryFormat.Pax, archiveStream, buffer);
+            }
         }
 
         // Asynchronously writes the current header as a PAX entry into the archive stream.
         // Makes sure to add the preceding exteded attributes entry before the actual entry.
         internal async Task WriteAsPaxAsync(Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
         {
+            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
             Debug.Assert(_typeFlag is not TarEntryType.GlobalExtendedAttributes);
             cancellationToken.ThrowIfCancellationRequested();
 
-            // First, we write the preceding extended attributes header
+            // First, we create the preceding extended attributes header
             TarHeader extendedAttributesHeader = new(TarEntryFormat.Pax);
-            // Fill the current header's dict
-            CollectExtendedAttributesFromStandardFieldsIfNeeded();
-            // And pass the attributes to the preceding extended attributes header for writing
-            await extendedAttributesHeader.WriteAsPaxExtendedAttributesAsync(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, cancellationToken).ConfigureAwait(false);
 
-            buffer.Span.Clear(); // Reset it to reuse it
-            // Second, we write this header as a normal one
-            await WriteAsAsync(TarEntryFormat.Pax, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
+            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
+            {
+                // Write the full entry header into a temporary stream, which will also collect the data length in the _size field
+                using MemoryStream tempStream = new();
+                // Don't advance the tempStream, instead, we will rewind it to the beginning for copying later
+                await WriteWithUnseekableDataStreamAsync(TarEntryFormat.Pax, tempStream, buffer, shouldAdvanceToEnd: false, cancellationToken).ConfigureAwait(false);
+                tempStream.Position = 0;
+                buffer.Span.Clear();
+
+                // If the data length is larger than it fits in the standard size field, it will get stored as an extended attribute
+                CollectExtendedAttributesFromStandardFieldsIfNeeded();
+
+                // Write the extended attributes entry into the archive first
+                await extendedAttributesHeader.WriteAsPaxExtendedAttributesAsync(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, cancellationToken).ConfigureAwait(false);
+                buffer.Span.Clear();
+
+                // And then write the stored entry into the archive
+                await tempStream.CopyToAsync(archiveStream, cancellationToken).ConfigureAwait(false);
+            }
+            else // Seek status of archive does not matter
+            {
+                _size = GetTotalDataBytesToWrite();
+                // Fill the current header's dict
+                CollectExtendedAttributesFromStandardFieldsIfNeeded();
+                // And pass the attributes to the preceding extended attributes header for writing
+                await extendedAttributesHeader.WriteAsPaxExtendedAttributesAsync(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, cancellationToken).ConfigureAwait(false);
+                buffer.Span.Clear(); // Reset it to reuse it
+
+                // Second, we write this header as a normal one
+                await WriteWithSeekableDataStreamAsync(TarEntryFormat.Pax, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
+            }
         }
 
         // Writes the current header as a Gnu entry into the archive stream.
         // Makes sure to add the preceding LongLink and/or LongPath entries if necessary, before the actual entry.
         internal void WriteAsGnu(Stream archiveStream, Span<byte> buffer)
         {
+            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
+
             // First, we determine if we need a preceding LongLink, and write it if needed
             if (_linkName != null && Encoding.UTF8.GetByteCount(_linkName) > FieldLengths.LinkName)
             {
                 TarHeader longLinkHeader = GetGnuLongMetadataHeader(TarEntryType.LongLink, _linkName);
-                longLinkHeader.WriteAs(TarEntryFormat.Gnu, archiveStream, buffer);
+                Debug.Assert(longLinkHeader._dataStream != null && longLinkHeader._dataStream.CanSeek); // We generate the long metadata data stream, should always be seekable
+                longLinkHeader.WriteWithSeekableDataStream(TarEntryFormat.Gnu, archiveStream, buffer);
                 buffer.Clear(); // Reset it to reuse it
             }
 
@@ -263,25 +364,35 @@ namespace System.Formats.Tar
             if (Encoding.UTF8.GetByteCount(_name) > FieldLengths.Name)
             {
                 TarHeader longPathHeader = GetGnuLongMetadataHeader(TarEntryType.LongPath, _name);
-                longPathHeader.WriteAs(TarEntryFormat.Gnu, archiveStream, buffer);
+                Debug.Assert(longPathHeader._dataStream != null && longPathHeader._dataStream.CanSeek); // We generate the long metadata data stream, should always be seekable
+                longPathHeader.WriteWithSeekableDataStream(TarEntryFormat.Gnu, archiveStream, buffer);
                 buffer.Clear(); // Reset it to reuse it
             }
 
             // Third, we write this header as a normal one
-            WriteAs(TarEntryFormat.Gnu, archiveStream, buffer);
+            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
+            {
+                WriteWithUnseekableDataStream(TarEntryFormat.Gnu, archiveStream, buffer, shouldAdvanceToEnd: true);
+            }
+            else // Seek status of archive does not matter
+            {
+                WriteWithSeekableDataStream(TarEntryFormat.Gnu, archiveStream, buffer);
+            }
         }
 
         // Writes the current header as a Gnu entry into the archive stream.
         // Makes sure to add the preceding LongLink and/or LongPath entries if necessary, before the actual entry.
         internal async Task WriteAsGnuAsync(Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
         {
+            Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
             cancellationToken.ThrowIfCancellationRequested();
 
             // First, we determine if we need a preceding LongLink, and write it if needed
             if (_linkName != null && Encoding.UTF8.GetByteCount(_linkName) > FieldLengths.LinkName)
             {
                 TarHeader longLinkHeader = GetGnuLongMetadataHeader(TarEntryType.LongLink, _linkName);
-                await longLinkHeader.WriteAsAsync(TarEntryFormat.Gnu, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
+                Debug.Assert(longLinkHeader._dataStream != null && longLinkHeader._dataStream.CanSeek); // We generate the long metadata data stream, should always be seekable
+                await longLinkHeader.WriteWithSeekableDataStreamAsync(TarEntryFormat.Gnu, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
                 buffer.Span.Clear(); // Reset it to reuse it
             }
 
@@ -289,37 +400,42 @@ namespace System.Formats.Tar
             if (Encoding.UTF8.GetByteCount(_name) > FieldLengths.Name)
             {
                 TarHeader longPathHeader = GetGnuLongMetadataHeader(TarEntryType.LongPath, _name);
-                await longPathHeader.WriteAsAsync(TarEntryFormat.Gnu, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
+                Debug.Assert(longPathHeader._dataStream != null && longPathHeader._dataStream.CanSeek); // We generate the long metadata data stream, should always be seekable
+                await longPathHeader.WriteWithSeekableDataStreamAsync(TarEntryFormat.Gnu, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
                 buffer.Span.Clear(); // Reset it to reuse it
             }
 
             // Third, we write this header as a normal one
-            await WriteAsAsync(TarEntryFormat.Gnu, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
+            if (archiveStream.CanSeek && _dataStream is { CanSeek: false })
+            {
+                await WriteWithUnseekableDataStreamAsync(TarEntryFormat.Gnu, archiveStream, buffer, shouldAdvanceToEnd: true, cancellationToken).ConfigureAwait(false);
+            }
+            else // Seek status of archive does not matter
+            {
+                await WriteWithSeekableDataStreamAsync(TarEntryFormat.Gnu, archiveStream, buffer, cancellationToken).ConfigureAwait(false);
+            }
         }
 
-        // Creates and returns a GNU long metadata header, with the specified long text written into its data stream.
+        // Creates and returns a GNU long metadata header, with the specified long text written into its data stream (seekable).
         private static TarHeader GetGnuLongMetadataHeader(TarEntryType entryType, string longText)
         {
             Debug.Assert(entryType is TarEntryType.LongPath or TarEntryType.LongLink);
 
-            TarHeader longMetadataHeader = new(TarEntryFormat.Gnu);
-
-            longMetadataHeader._name = GnuLongMetadataName; // Same name for both longpath or longlink
-            longMetadataHeader._mode = TarHelpers.GetDefaultMode(entryType);
-            longMetadataHeader._uid = 0;
-            longMetadataHeader._gid = 0;
-            longMetadataHeader._mTime = DateTimeOffset.MinValue; // 0
-            longMetadataHeader._typeFlag = entryType;
-            longMetadataHeader._dataStream = new MemoryStream(Encoding.UTF8.GetBytes(longText));
-
-            return longMetadataHeader;
+            return new(TarEntryFormat.Gnu)
+            {
+                _name = GnuLongMetadataName, // Same name for both longpath or longlink
+                _mode = TarHelpers.GetDefaultMode(entryType),
+                _uid = 0,
+                _gid = 0,
+                _mTime = DateTimeOffset.MinValue, // 0
+                _typeFlag = entryType,
+                _dataStream = new MemoryStream(Encoding.UTF8.GetBytes(longText))
+            };
         }
 
         // Shared checksum and data length calculations for GNU entry writing.
-        private void WriteGnuFieldsToBuffer(long size, Span<byte> buffer)
+        private void WriteGnuFieldsToBuffer(Span<byte> buffer)
         {
-            _size = size;
-
             int tmpChecksum = WriteName(buffer);
             tmpChecksum += WriteCommonFields(buffer, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Gnu, _typeFlag));
             tmpChecksum += WriteGnuMagicAndVersion(buffer);
@@ -333,7 +449,8 @@ namespace System.Formats.Tar
         private void WriteAsPaxExtendedAttributes(Stream archiveStream, Span<byte> buffer, Dictionary<string, string> extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber)
         {
             WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes);
-            WriteAs(TarEntryFormat.Pax, archiveStream, buffer);
+            Debug.Assert(_dataStream == null || (extendedAttributes.Count > 0 && _dataStream.CanSeek)); // We generate the extended attributes data stream, should always be seekable
+            WriteWithSeekableDataStream(TarEntryFormat.Pax, archiveStream, buffer);
         }
 
         // Asynchronously writes the current header as a PAX Extended Attributes entry into the archive stream and returns the value of the final checksum.
@@ -341,7 +458,8 @@ namespace System.Formats.Tar
         {
             cancellationToken.ThrowIfCancellationRequested();
             WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes);
-            return WriteAsAsync(TarEntryFormat.Pax, archiveStream, buffer, cancellationToken);
+            Debug.Assert(_dataStream == null || (extendedAttributes.Count > 0 && _dataStream.CanSeek)); // We generate the extended attributes data stream, should always be seekable
+            return WriteWithSeekableDataStreamAsync(TarEntryFormat.Pax, archiveStream, buffer, cancellationToken);
         }
 
         // Initializes the name, mode and type flag of a PAX extended attributes entry.
@@ -359,9 +477,8 @@ namespace System.Formats.Tar
         }
 
         // Shared checksum and data length calculations for PAX entry writing.
-        private void WritePaxFieldsToBuffer(long size, Span<byte> buffer)
+        private void WritePaxFieldsToBuffer(Span<byte> buffer)
         {
-            _size = size;
             int tmpChecksum = WriteName(buffer);
             tmpChecksum += WriteCommonFields(buffer, TarHelpers.GetCorrectTypeFlagForFormat(TarEntryFormat.Pax, _typeFlag));
             tmpChecksum += WritePosixMagicAndVersion(buffer);
@@ -371,21 +488,21 @@ namespace System.Formats.Tar
         }
 
         // Writes the format-specific fields of the current entry, as well as the entry data length, into the specified buffer.
-        private void WriteFieldsToBuffer(TarEntryFormat format, long bytesToWrite, Span<byte> buffer)
+        private void WriteFieldsToBuffer(TarEntryFormat format, Span<byte> buffer)
         {
             switch (format)
             {
                 case TarEntryFormat.V7:
-                    WriteV7FieldsToBuffer(bytesToWrite, buffer);
+                    WriteV7FieldsToBuffer(buffer);
                     break;
                 case TarEntryFormat.Ustar:
-                    WriteUstarFieldsToBuffer(bytesToWrite, buffer);
+                    WriteUstarFieldsToBuffer(buffer);
                     break;
                 case TarEntryFormat.Pax:
-                    WritePaxFieldsToBuffer(bytesToWrite, buffer);
+                    WritePaxFieldsToBuffer(buffer);
                     break;
                 case TarEntryFormat.Gnu:
-                    WriteGnuFieldsToBuffer(bytesToWrite, buffer);
+                    WriteGnuFieldsToBuffer(buffer);
                     break;
             }
         }
@@ -514,6 +631,7 @@ namespace System.Formats.Tar
                 }
                 else
                 {
+                    // No writing, just verifications
                     Debug.Assert(_typeFlag is not TarEntryType.ExtendedAttributes and not TarEntryType.GlobalExtendedAttributes);
                     Debug.Assert(Convert.ToInt64(ExtendedAttributes[PaxEaSize]) > TarHelpers.MaxSizeLength);
                 }
@@ -548,12 +666,13 @@ namespace System.Formats.Tar
 
         // Calculates how many data bytes should be written, depending on the position pointer of the stream.
         // Only works if the stream is seekable.
-        private long GetTotalDataBytesToWrite()
+        public long GetTotalDataBytesToWrite()
         {
             if (_dataStream == null)
             {
                 return 0;
             }
+            Debug.Assert(_dataStream.CanSeek);
 
             long length = _dataStream.Length;
             long position = _dataStream.Position;
@@ -646,16 +765,16 @@ namespace System.Formats.Tar
         }
 
         // Writes the current header's data stream into the archive stream.
-        private static void WriteData(Stream archiveStream, Stream dataStream, long actualLength)
+        private void WriteData(Stream archiveStream, Stream dataStream)
         {
             dataStream.CopyTo(archiveStream); // The data gets copied from the current position
-            WriteEmptyPadding(archiveStream, actualLength);
+            WriteEmptyPadding(archiveStream);
         }
 
         // Calculates the padding for the current entry and writes it after the data.
-        private static void WriteEmptyPadding(Stream archiveStream, long actualLength)
+        private void WriteEmptyPadding(Stream archiveStream)
         {
-            int paddingAfterData = TarHelpers.CalculatePadding(actualLength);
+            int paddingAfterData = TarHelpers.CalculatePadding(_size);
             if (paddingAfterData != 0)
             {
                 Debug.Assert(paddingAfterData <= TarHelpers.RecordSize);
@@ -669,9 +788,9 @@ namespace System.Formats.Tar
         }
 
         // Calculates the padding for the current entry and asynchronously writes it after the data.
-        private static ValueTask WriteEmptyPaddingAsync(Stream archiveStream, long actualLength, CancellationToken cancellationToken)
+        private ValueTask WriteEmptyPaddingAsync(Stream archiveStream, CancellationToken cancellationToken)
         {
-            int paddingAfterData = TarHelpers.CalculatePadding(actualLength);
+            int paddingAfterData = TarHelpers.CalculatePadding(_size);
             if (paddingAfterData != 0)
             {
                 Debug.Assert(paddingAfterData <= TarHelpers.RecordSize);
@@ -684,13 +803,13 @@ namespace System.Formats.Tar
         }
 
         // Asynchronously writes the current header's data stream into the archive stream.
-        private static async Task WriteDataAsync(Stream archiveStream, Stream dataStream, long actualLength, CancellationToken cancellationToken)
+        private async Task WriteDataAsync(Stream archiveStream, Stream dataStream, CancellationToken cancellationToken)
         {
             cancellationToken.ThrowIfCancellationRequested();
 
             await dataStream.CopyToAsync(archiveStream, cancellationToken).ConfigureAwait(false); // The data gets copied from the current position
 
-            int paddingAfterData = TarHelpers.CalculatePadding(actualLength);
+            int paddingAfterData = TarHelpers.CalculatePadding(_size);
             if (paddingAfterData != 0)
             {
                 byte[] buffer = ArrayPool<byte>.Shared.Rent(paddingAfterData);
@@ -702,7 +821,8 @@ namespace System.Formats.Tar
             }
         }
 
-        // Dumps into the archive stream an extended attribute entry containing metadata of the entry it precedes.
+        // Generates a data stream (seekable) containing the extended attribute metadata of the entry it precedes.
+        // Returns a null stream if the extended attributes dictionary is empty.
         private static MemoryStream? GenerateExtendedAttributesDataStream(Dictionary<string, string> extendedAttributes)
         {
             MemoryStream? dataStream = null;
@@ -822,7 +942,7 @@ namespace System.Formats.Tar
         // The checksum accumulator first adds up the byte values of eight space chars, then the final number
         // is written on top of those spaces on the specified span as ascii.
         // At the end, it's saved in the header field and the final value returned.
-        internal static int WriteChecksum(int checksum, Span<byte> buffer)
+        private static int WriteChecksum(int checksum, Span<byte> buffer)
         {
             // The checksum field is also counted towards the total sum
             // but as an array filled with spaces
@@ -903,7 +1023,7 @@ namespace System.Formats.Tar
         }
 
         // Writes the specified decimal number as a right-aligned octal number and returns its checksum.
-        internal static int FormatOctal(long value, Span<byte> destination)
+        private static int FormatOctal(long value, Span<byte> destination)
         {
             ulong remaining = (ulong)value;
             Span<byte> digits = stackalloc byte[32]; // longer than any possible octal formatting of a ulong
index 13654ea..df77341 100644 (file)
@@ -283,8 +283,12 @@ namespace System.Formats.Tar
 
             switch (entry.Format)
             {
-                case TarEntryFormat.V7 or TarEntryFormat.Ustar:
-                    entry._header.WriteAs(entry.Format, _archiveStream, buffer);
+                case TarEntryFormat.V7:
+                    entry._header.WriteAsV7(_archiveStream, buffer);
+                    break;
+
+                case TarEntryFormat.Ustar:
+                    entry._header.WriteAsUstar(_archiveStream, buffer);
                     break;
 
                 case TarEntryFormat.Pax:
@@ -321,7 +325,8 @@ namespace System.Formats.Tar
 
             Task task = entry.Format switch
             {
-                TarEntryFormat.V7 or TarEntryFormat.Ustar => entry._header.WriteAsAsync(entry.Format, _archiveStream, buffer, cancellationToken),
+                TarEntryFormat.V7 => entry._header.WriteAsV7Async(_archiveStream, buffer, cancellationToken),
+                TarEntryFormat.Ustar => entry._header.WriteAsUstarAsync(_archiveStream, buffer, cancellationToken),
                 TarEntryFormat.Pax when entry._header._typeFlag is TarEntryType.GlobalExtendedAttributes => entry._header.WriteAsPaxGlobalExtendedAttributesAsync(_archiveStream, buffer, _nextGlobalExtendedAttributesEntryNumber++, cancellationToken),
                 TarEntryFormat.Pax => entry._header.WriteAsPaxAsync(_archiveStream, buffer, cancellationToken),
                 TarEntryFormat.Gnu => entry._header.WriteAsGnuAsync(_archiveStream, buffer, cancellationToken),
diff --git a/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs b/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs
new file mode 100644 (file)
index 0000000..1fa1c68
--- /dev/null
@@ -0,0 +1,93 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.IO;
+using Xunit;
+
+namespace System.Formats.Tar.Tests;
+
+[OuterLoop]
+[Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time
+public class ManualTests : TarTestsBase
+{
+    public static bool ManualTestsEnabled => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MANUAL_TESTS"));
+
+    public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryData()
+    {
+        foreach (bool unseekableStream in new[] { false, true })
+        {
+            foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax })
+            {
+                yield return new object[] { entryFormat, LegacyMaxFileSize, unseekableStream };
+            }
+
+            // Pax supports unlimited size files.
+            yield return new object[] { TarEntryFormat.Pax, LegacyMaxFileSize + 1, unseekableStream };
+        }
+    }
+
+    [ConditionalTheory(nameof(ManualTestsEnabled))]
+    [MemberData(nameof(WriteEntry_LongFileSize_TheoryData))]
+    [SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")]
+    public void WriteEntry_LongFileSize(TarEntryFormat entryFormat, long size, bool unseekableStream)
+    {
+        // Write archive with a 8 Gb long entry.
+        using FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose });
+        Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile;
+
+        using (TarWriter writer = new(s, leaveOpen: true))
+        {
+            TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo");
+            writeEntry.DataStream = new SimulatedDataStream(size);
+            writer.WriteEntry(writeEntry);
+        }
+
+        tarFile.Position = 0;
+
+        // Read archive back.
+        using TarReader reader = new TarReader(s);
+        TarEntry entry = reader.GetNextEntry();
+        Assert.Equal(size, entry.Length);
+
+        Stream dataStream = entry.DataStream;
+        Assert.Equal(size, dataStream.Length);
+        Assert.Equal(0, dataStream.Position);
+
+        ReadOnlySpan<byte> dummyData = SimulatedDataStream.DummyData.Span;
+
+        // Read the first bytes.
+        Span<byte> buffer = new byte[dummyData.Length];
+        Assert.Equal(buffer.Length, dataStream.Read(buffer));
+        AssertExtensions.SequenceEqual(dummyData, buffer);
+        Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct.
+        buffer.Clear();
+
+        // Read the last bytes.
+        long dummyDataOffset = size - dummyData.Length - 1;
+        if (dataStream.CanSeek)
+        {
+            Assert.False(unseekableStream);
+            dataStream.Seek(dummyDataOffset, SeekOrigin.Begin);
+        }
+        else
+        {
+            Assert.True(unseekableStream);
+            Span<byte> seekBuffer = new byte[4_096];
+
+            while (dataStream.Position < dummyDataOffset)
+            {
+                int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position);
+                int res = dataStream.Read(seekBuffer.Slice(0, bufSize));
+                Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong");
+            }
+        }
+
+        Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct.
+        Assert.Equal(buffer.Length, dataStream.Read(buffer));
+        AssertExtensions.SequenceEqual(dummyData, buffer);
+        Assert.Equal(size, dataStream.Position);
+
+        Assert.Null(reader.GetNextEntry());
+    }
+}
diff --git a/src/libraries/System.Formats.Tar/tests/Manual/ManualTestsAsync.cs b/src/libraries/System.Formats.Tar/tests/Manual/ManualTestsAsync.cs
new file mode 100644 (file)
index 0000000..5262c71
--- /dev/null
@@ -0,0 +1,82 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace System.Formats.Tar.Tests;
+
+[OuterLoop]
+[Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time
+public class ManualTestsAsync : TarTestsBase
+{
+    public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryDataAsync()
+        // Fixes error xUnit1015: MemberData needs to be in the same class
+        => ManualTests.WriteEntry_LongFileSize_TheoryData();
+
+    [ConditionalTheory(nameof(ManualTests.ManualTestsEnabled))]
+    [MemberData(nameof(WriteEntry_LongFileSize_TheoryDataAsync))]
+    [SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")]
+    public async Task WriteEntry_LongFileSizeAsync(TarEntryFormat entryFormat, long size, bool unseekableStream)
+    {
+        // Write archive with a 8 Gb long entry.
+        await using FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose });
+        Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile;
+
+        await using (TarWriter writer = new(s, leaveOpen: true))
+        {
+            TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo");
+            writeEntry.DataStream = new SimulatedDataStream(size);
+            await writer.WriteEntryAsync(writeEntry);
+        }
+
+        tarFile.Position = 0;
+
+        // Read the archive back.
+        await using TarReader reader = new TarReader(s);
+        TarEntry entry = await reader.GetNextEntryAsync();
+        Assert.Equal(size, entry.Length);
+
+        Stream dataStream = entry.DataStream;
+        Assert.Equal(size, dataStream.Length);
+        Assert.Equal(0, dataStream.Position);
+
+        ReadOnlyMemory<byte> dummyData = SimulatedDataStream.DummyData;
+
+        // Read the first bytes.
+        byte[] buffer = new byte[dummyData.Length];
+        Assert.Equal(buffer.Length, dataStream.Read(buffer));
+        AssertExtensions.SequenceEqual(dummyData.Span, buffer);
+        Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct.
+        buffer.AsSpan().Clear();
+
+        // Read the last bytes.
+        long dummyDataOffset = size - dummyData.Length - 1;
+        if (dataStream.CanSeek)
+        {
+            Assert.False(unseekableStream);
+            dataStream.Seek(dummyDataOffset, SeekOrigin.Begin);
+        }
+        else
+        {
+            Assert.True(unseekableStream);
+            Memory<byte> seekBuffer = new byte[4_096];
+
+            while (dataStream.Position < dummyDataOffset)
+            {
+                int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position);
+                int res = await dataStream.ReadAsync(seekBuffer.Slice(0, bufSize));
+                Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong");
+            }
+        }
+
+        Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct.
+        Assert.Equal(buffer.Length, dataStream.Read(buffer));
+        AssertExtensions.SequenceEqual(dummyData.Span, buffer);
+        Assert.Equal(size, dataStream.Position);
+
+        Assert.Null(await reader.GetNextEntryAsync());
+    }
+}
diff --git a/src/libraries/System.Formats.Tar/tests/Manual/System.Formats.Tar.Manual.Tests.csproj b/src/libraries/System.Formats.Tar/tests/Manual/System.Formats.Tar.Manual.Tests.csproj
new file mode 100644 (file)
index 0000000..99adaab
--- /dev/null
@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
+    <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="ManualTests.cs" />
+    <Compile Include="ManualTestsAsync.cs" />
+    <Compile Include="..\TarTestsBase.cs" />
+    <Compile Include="..\SimulatedDataStream.cs" />
+    <Compile Include="$(CommonTestPath)TestUtilities\System\DisableParallelization.cs" Link="Common\TestUtilities\System\DisableParallelization.cs" />
+    <Compile Include="$(CommonTestPath)System\IO\TempDirectory.cs" Link="Common\System\IO\TempDirectory.cs" />
+    <Compile Include="$(CommonTestPath)System\IO\WrappedStream.cs" Link="Common\System\IO\WrappedStream.cs" />
+  </ItemGroup>
+</Project>
index fd3f3ba..f3a341d 100644 (file)
     <Compile Include="TarTestsBase.Ustar.cs" />
     <Compile Include="TarTestsBase.V7.cs" />
     <Compile Include="TarWriter\TarWriter.WriteEntry.Entry.Roundtrip.Tests.cs" />
-    <Compile Include="TarWriter\TarWriter.WriteEntry.LongFile.Tests.cs" />
     <Compile Include="TarWriter\TarWriter.WriteEntryAsync.File.Tests.cs" />
     <Compile Include="TarWriter\TarWriter.WriteEntry.Base.cs" />
-    <Compile Include="TarWriter\TarWriter.WriteEntryAsync.LongFile.Tests.cs" />
     <Compile Include="TarWriter\TarWriter.WriteEntryAsync.Tests.cs" />
     <Compile Include="TarWriter\TarWriter.WriteEntryAsync.Entry.Roundtrip.Tests.cs" />
     <Compile Include="TarWriter\TarWriter.WriteEntryAsync.Entry.Ustar.Tests.cs" />
@@ -74,7 +72,6 @@
     <Compile Include="$(CommonPath)DisableRuntimeMarshalling.cs" Link="Common\DisableRuntimeMarshalling.cs" />
     <Compile Include="$(CommonTestPath)System\IO\ReparsePointUtilities.cs" Link="Common\System\IO\ReparsePointUtilities.cs" />
     <Compile Include="$(CommonTestPath)System\IO\WrappedStream.cs" Link="Common\System\IO\WrappedStream.cs" />
-    <Compile Include="$(CommonTestPath)TestUtilities\System\DisableParallelization.cs" Link="Common\TestUtilities\System\DisableParallelization.cs" />
   </ItemGroup>
   <!-- Windows specific files -->
   <ItemGroup Condition="'$(TargetPlatformIdentifier)' == 'windows'">
diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.LongFile.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.LongFile.Tests.cs
deleted file mode 100644 (file)
index 1368f45..0000000
+++ /dev/null
@@ -1,92 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-using System.Collections.Generic;
-using System.IO;
-using Xunit;
-
-namespace System.Formats.Tar.Tests
-{
-    [OuterLoop]
-    [Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time
-    public class TarWriter_WriteEntry_LongFile_Tests : TarTestsBase
-    {
-        public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryData()
-        {
-            foreach (bool unseekableStream in new[] { false, true })
-            {
-                foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax })
-                {
-                    yield return new object[] { entryFormat, LegacyMaxFileSize, unseekableStream };
-                }
-
-                // Pax supports unlimited size files.
-                yield return new object[] { TarEntryFormat.Pax, LegacyMaxFileSize + 1, unseekableStream };
-            }
-        }
-
-        [Theory]
-        [MemberData(nameof(WriteEntry_LongFileSize_TheoryData))]
-        [SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")]
-        public void WriteEntry_LongFileSize(TarEntryFormat entryFormat, long size, bool unseekableStream)
-        {
-            // Write archive with a 8 Gb long entry.
-            using FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose });
-            Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile;
-
-            using (TarWriter writer = new(s, leaveOpen: true))
-            {
-                TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo");
-                writeEntry.DataStream = new SimulatedDataStream(size);
-                writer.WriteEntry(writeEntry);
-            }
-
-            tarFile.Position = 0;
-
-            // Read archive back.
-            using TarReader reader = new TarReader(s);
-            TarEntry entry = reader.GetNextEntry();
-            Assert.Equal(size, entry.Length);
-
-            Stream dataStream = entry.DataStream;
-            Assert.Equal(size, dataStream.Length);
-            Assert.Equal(0, dataStream.Position);
-
-            ReadOnlySpan<byte> dummyData = SimulatedDataStream.DummyData.Span;
-
-            // Read the first bytes.
-            Span<byte> buffer = new byte[dummyData.Length];
-            Assert.Equal(buffer.Length, dataStream.Read(buffer));
-            AssertExtensions.SequenceEqual(dummyData, buffer);
-            Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct.
-            buffer.Clear();
-
-            // Read the last bytes.
-            long dummyDataOffset = size - dummyData.Length - 1;
-            if (dataStream.CanSeek)
-            {
-                Assert.False(unseekableStream);
-                dataStream.Seek(dummyDataOffset, SeekOrigin.Begin);
-            }
-            else
-            {
-                Assert.True(unseekableStream);
-                Span<byte> seekBuffer = new byte[4_096];
-
-                while (dataStream.Position < dummyDataOffset)
-                {
-                    int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position);
-                    int res = dataStream.Read(seekBuffer.Slice(0, bufSize));
-                    Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong");
-                }
-            }
-
-            Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct.
-            Assert.Equal(buffer.Length, dataStream.Read(buffer));
-            AssertExtensions.SequenceEqual(dummyData, buffer);
-            Assert.Equal(size, dataStream.Position);
-
-            Assert.Null(reader.GetNextEntry());
-        }
-    }
-}
diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.LongFile.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.LongFile.Tests.cs
deleted file mode 100644 (file)
index 837da5f..0000000
+++ /dev/null
@@ -1,82 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-using System.Collections.Generic;
-using System.IO;
-using System.Threading.Tasks;
-using Xunit;
-
-namespace System.Formats.Tar.Tests
-{
-    [OuterLoop]
-    [Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time
-    public class TarWriter_WriteEntryAsync_LongFile_Tests : TarTestsBase
-    {
-        public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryDataAsync()
-            => TarWriter_WriteEntry_LongFile_Tests.WriteEntry_LongFileSize_TheoryData();
-
-        [Theory]
-        [MemberData(nameof(WriteEntry_LongFileSize_TheoryDataAsync))]
-        [SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")]
-        public async Task WriteEntry_LongFileSizeAsync(TarEntryFormat entryFormat, long size, bool unseekableStream)
-        {
-            // Write archive with a 8 Gb long entry.
-            await using FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose });
-            Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile;
-
-            await using (TarWriter writer = new(s, leaveOpen: true))
-            {
-                TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo");
-                writeEntry.DataStream = new SimulatedDataStream(size);
-                await writer.WriteEntryAsync(writeEntry);
-            }
-
-            tarFile.Position = 0;
-
-            // Read the archive back.
-            await using TarReader reader = new TarReader(s);
-            TarEntry entry = await reader.GetNextEntryAsync();
-            Assert.Equal(size, entry.Length);
-
-            Stream dataStream = entry.DataStream;
-            Assert.Equal(size, dataStream.Length);
-            Assert.Equal(0, dataStream.Position);
-
-            ReadOnlyMemory<byte> dummyData = SimulatedDataStream.DummyData;
-
-            // Read the first bytes.
-            byte[] buffer = new byte[dummyData.Length];
-            Assert.Equal(buffer.Length, dataStream.Read(buffer));
-            AssertExtensions.SequenceEqual(dummyData.Span, buffer);
-            Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct.
-            buffer.AsSpan().Clear();
-
-            // Read the last bytes.
-            long dummyDataOffset = size - dummyData.Length - 1;
-            if (dataStream.CanSeek)
-            {
-                Assert.False(unseekableStream);
-                dataStream.Seek(dummyDataOffset, SeekOrigin.Begin);
-            }
-            else
-            {
-                Assert.True(unseekableStream);
-                Memory<byte> seekBuffer = new byte[4_096];
-
-                while (dataStream.Position < dummyDataOffset)
-                {
-                    int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position);
-                    int res = await dataStream.ReadAsync(seekBuffer.Slice(0, bufSize));
-                    Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong");
-                }
-            }
-
-            Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct.
-            Assert.Equal(buffer.Length, dataStream.Read(buffer));
-            AssertExtensions.SequenceEqual(dummyData.Span, buffer);
-            Assert.Equal(size, dataStream.Position);
-
-            Assert.Null(await reader.GetNextEntryAsync());
-        }
-    }
-}