Add System.Resources.Extensions (dotnet/corefxdotnet/coreclr#36906)
authorEric StJohn <ericstj@microsoft.com>
Fri, 26 Apr 2019 05:05:19 +0000 (22:05 -0700)
committerJan Kotas <jkotas@microsoft.com>
Sat, 27 Apr 2019 12:09:40 +0000 (05:09 -0700)
* Add System.Resources.Binary.Reader|Writer

* Fix ResourceWriter tests

* Test fixes and PR feedback

* More test fixes

* Add packages for System.Resources.Binary.*

* Suppress duplicate types in System.Resources.Binary.*

* Test refactoring and adding RuntimeResourceSet

It turns out me must have our own ResourceSet since the CoreLib resource set doesn't
expose a constructor that takes an IResourceReader.

I've shared the code since it does a bit of non-trivial caching.

* Don't use auto-property initializers for platfrom sensitive test data

For some reason I thought these lazy-initialized the properties but they don't.

As a result we were hitting the platform sensitive code even when we never
called the getter.  Switch to an expression instead.

* Only use Drawing converters on Windows

* Fix test failures

* Don't leak System.Private.CoreLib into resources

* Make sure RuntimeResourceSet doesn't call ResourceReader(IResourceReader)

* WIP

* Rename types in System.Resources.Extensions

Leave RuntimeResourceSet internal as it doesn't need to be public.

* Update packages

* Respond to API review feedback

Remove abstraction for ResourceReader/Writer: just reuse the source.

Remove non-essential members.

* Clean up

* Further cleanup

* Further cleanup

* Review feedback

* Ensure we have stable type names in Resources.Extensions

We don't want to use the runtime type identity when doing type checks or writing
types as this may change.  Instead, check-in hard-coded strings that match the
type identity.

Add a test case to ensure the resources we generate match what we
test the reader against in the resource manager case (also to track any
change to the binary format).

Signed-off-by: dotnet-bot <dotnet-bot@microsoft.com>
Commit migrated from https://github.com/dotnet/coreclr/commit/ae6c7a9a7ddfa43f30450fa4085d952a1aaf3723

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems
src/libraries/System.Private.CoreLib/src/System/IO/PinnedBufferMemoryStream.cs
src/libraries/System.Private.CoreLib/src/System/Resources/FastResourceComparer.cs
src/libraries/System.Private.CoreLib/src/System/Resources/ResourceReader.Core.cs [new file with mode: 0644]
src/libraries/System.Private.CoreLib/src/System/Resources/ResourceReader.cs
src/libraries/System.Private.CoreLib/src/System/Resources/RuntimeResourceSet.cs

index 7e1cb0d..4562862 100644 (file)
     <Compile Include="$(MSBuildThisFileDirectory)System\Resources\ResourceFallbackManager.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Resources\ResourceManager.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Resources\ResourceReader.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Resources\ResourceReader.Core.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Resources\ResourceSet.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Resources\ResourceTypeCode.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Resources\RuntimeResourceSet.cs" />
index 7c636b9..2589eee 100644 (file)
@@ -39,9 +39,11 @@ namespace System.IO
                 Initialize(ptr, len, len, FileAccess.Read);
         }
 
+#if !netstandard
         public override int Read(Span<byte> buffer) => ReadCore(buffer);
 
         public override void Write(ReadOnlySpan<byte> buffer) => WriteCore(buffer);
+#endif
 
         ~PinnedBufferMemoryStream()
         {
index 51058c5..5f83b77 100644 (file)
@@ -15,7 +15,6 @@
 ** 
 ===========================================================*/
 
-using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics;
diff --git a/src/libraries/System.Private.CoreLib/src/System/Resources/ResourceReader.Core.cs b/src/libraries/System.Private.CoreLib/src/System/Resources/ResourceReader.Core.cs
new file mode 100644 (file)
index 0000000..9d25e45
--- /dev/null
@@ -0,0 +1,162 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace System.Resources
+{
+    partial class ResourceReader
+    {
+        private readonly bool _permitDeserialization;  // can deserialize BinaryFormatted resources
+        private object? _binaryFormatter; // binary formatter instance to use for deserializing
+
+        // statics used to dynamically call into BinaryFormatter
+        // When successfully located s_binaryFormatterType will point to the BinaryFormatter type
+        // and s_deserializeMethod will point to an unbound delegate to the deserialize method.
+        private static Type s_binaryFormatterType;
+        private static Func<object?, Stream, object> s_deserializeMethod;
+
+        // This is the constructor the RuntimeResourceSet calls,
+        // passing in the stream to read from and the RuntimeResourceSet's 
+        // internal hash table (hash table of names with file offsets
+        // and values, coupled to this ResourceReader).
+        internal ResourceReader(Stream stream, Dictionary<string, ResourceLocator> resCache, bool permitDeserialization)
+        {
+            Debug.Assert(stream != null, "Need a stream!");
+            Debug.Assert(stream.CanRead, "Stream should be readable!");
+            Debug.Assert(resCache != null, "Need a Dictionary!");
+
+            _resCache = resCache;
+            _store = new BinaryReader(stream, Encoding.UTF8);
+
+            _ums = stream as UnmanagedMemoryStream;
+
+            _permitDeserialization = permitDeserialization;
+
+            ReadResources();
+        }
+
+        private object DeserializeObject(int typeIndex)
+        {
+            if (!_permitDeserialization)
+            {
+                throw new NotSupportedException(SR.NotSupported_ResourceObjectSerialization);
+            }
+
+            if (_binaryFormatter == null)
+            {
+                InitializeBinaryFormatter();
+            }
+
+            Type type = FindType(typeIndex);
+
+            object graph = s_deserializeMethod(_binaryFormatter, _store.BaseStream);
+
+            // guard against corrupted resources
+            if (graph.GetType() != type)
+                throw new BadImageFormatException(SR.Format(SR.BadImageFormat_ResType_SerBlobMismatch, type.FullName, graph.GetType().FullName));
+
+            return graph;
+        }
+
+        private void InitializeBinaryFormatter()
+        {
+            LazyInitializer.EnsureInitialized(ref s_binaryFormatterType, () =>
+                Type.GetType("System.Runtime.Serialization.Formatters.Binary.BinaryFormatter, System.Runtime.Serialization.Formatters, Version=0.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
+                throwOnError: true));
+
+            LazyInitializer.EnsureInitialized(ref s_deserializeMethod, () =>
+            {
+                MethodInfo binaryFormatterDeserialize = s_binaryFormatterType.GetMethod("Deserialize", new Type[] { typeof(Stream) });
+
+                // create an unbound delegate that can accept a BinaryFormatter instance as object
+                return (Func<object?, Stream, object>)typeof(ResourceReader)
+                        .GetMethod(nameof(CreateUntypedDelegate), BindingFlags.NonPublic | BindingFlags.Static)
+                        .MakeGenericMethod(s_binaryFormatterType)
+                        .Invoke(null, new object[] { binaryFormatterDeserialize });
+            });
+
+            _binaryFormatter = Activator.CreateInstance(s_binaryFormatterType);
+        }
+
+        // generic method that we specialize at runtime once we've loaded the BinaryFormatter type
+        // permits creating an unbound delegate so that we can avoid reflection after the initial
+        // lightup code completes.
+        private static Func<object, Stream, object> CreateUntypedDelegate<TInstance>(MethodInfo method)
+        {
+            Func<TInstance, Stream, object> typedDelegate = (Func<TInstance, Stream, object>)Delegate.CreateDelegate(typeof(Func<TInstance, Stream, object>), null, method);
+
+            return (obj, stream) => typedDelegate((TInstance)obj, stream);
+        }
+
+        private bool ValidateReaderType(string readerType)
+        {
+            return ResourceManager.IsDefaultType(readerType, ResourceManager.ResReaderTypeName);
+        }
+
+        public void GetResourceData(string resourceName, out string resourceType, out byte[] resourceData)
+        {
+            if (resourceName == null)
+                throw new ArgumentNullException(nameof(resourceName));
+            if (_resCache == null)
+                throw new InvalidOperationException(SR.ResourceReaderIsClosed);
+
+            // Get the type information from the data section.  Also,
+            // sort all of the data section's indexes to compute length of
+            // the serialized data for this type (making sure to subtract
+            // off the length of the type code).
+            int[] sortedDataPositions = new int[_numResources];
+            int dataPos = FindPosForResource(resourceName);
+            if (dataPos == -1)
+            {
+                throw new ArgumentException(SR.Format(SR.Arg_ResourceNameNotExist, resourceName));
+            }
+
+            lock (this)
+            {
+                // Read all the positions of data within the data section.
+                for (int i = 0; i < _numResources; i++)
+                {
+                    _store.BaseStream.Position = _nameSectionOffset + GetNamePosition(i);
+                    // Skip over name of resource
+                    int numBytesToSkip = _store.Read7BitEncodedInt();
+                    if (numBytesToSkip < 0)
+                    {
+                        throw new FormatException(SR.Format(SR.BadImageFormat_ResourcesNameInvalidOffset, numBytesToSkip));
+                    }
+                    _store.BaseStream.Position += numBytesToSkip;
+
+                    int dPos = _store.ReadInt32();
+                    if (dPos < 0 || dPos >= _store.BaseStream.Length - _dataSectionOffset)
+                    {
+                        throw new FormatException(SR.Format(SR.BadImageFormat_ResourcesDataInvalidOffset, dPos));
+                    }
+                    sortedDataPositions[i] = dPos;
+                }
+                Array.Sort(sortedDataPositions);
+
+                int index = Array.BinarySearch(sortedDataPositions, dataPos);
+                Debug.Assert(index >= 0 && index < _numResources, "Couldn't find data position within sorted data positions array!");
+                long nextData = (index < _numResources - 1) ? sortedDataPositions[index + 1] + _dataSectionOffset : _store.BaseStream.Length;
+                int len = (int)(nextData - (dataPos + _dataSectionOffset));
+                Debug.Assert(len >= 0 && len <= (int)_store.BaseStream.Length - dataPos + _dataSectionOffset, "Length was negative or outside the bounds of the file!");
+
+                // Read type code then byte[]
+                _store.BaseStream.Position = _dataSectionOffset + dataPos;
+                ResourceTypeCode typeCode = (ResourceTypeCode)_store.Read7BitEncodedInt();
+                if (typeCode < 0 || typeCode >= ResourceTypeCode.StartOfUserTypes + _typeTable.Length)
+                {
+                    throw new BadImageFormatException(SR.BadImageFormat_InvalidType);
+                }
+                resourceType = TypeNameFromTypeCode(typeCode);
+
+                // The length must be adjusted to subtract off the number 
+                // of bytes in the 7 bit encoded type code.
+                len -= (int)(_store.BaseStream.Position - (_dataSectionOffset + dataPos));
+                byte[] bytes = _store.ReadBytes(len);
+                if (bytes.Length != len)
+                    throw new FormatException(SR.BadImageFormat_ResourceNameCorrupted);
+                resourceData = bytes;
+            }
+        }
+    }
+}
index d854550..510082d 100644 (file)
 ===========================================================*/
 
 namespace System.Resources
+#if RESOURCES_EXTENSIONS
+    .Extensions
+#endif
 {
     using System;
     using System.IO;
     using System.Text;
     using System.Collections;
     using System.Collections.Generic;
-    using System.Reflection;
-    using System.Security;
-    using System.Globalization;
-    using System.Configuration.Assemblies;
-    using System.Runtime.Versioning;
     using System.Diagnostics;
-    using System.Diagnostics.Contracts;
-    using System.Threading;
 
+#if RESOURCES_EXTENSIONS
+    using ResourceReader = DeserializingResourceReader;
+#endif
     // Provides the default implementation of IResourceReader, reading
     // .resources file from the system default binary format.  This class
     // can be treated as an enumerator once.
@@ -72,7 +71,13 @@ namespace System.Resources
         }
     }
 
-    public sealed class ResourceReader : IResourceReader
+    public sealed partial class
+#if RESOURCES_EXTENSIONS
+        DeserializingResourceReader
+#else
+        ResourceReader
+#endif
+        : IResourceReader
     {
         // A reasonable default buffer size for reading from files, especially
         // when we will likely be seeking frequently.  Could be smaller, but does
@@ -100,15 +105,6 @@ namespace System.Resources
         private int[] _typeNamePositions = null!;  // To delay initialize type table
         private int _numResources;    // Num of resources files, in case arrays aren't allocated.
 
-        private readonly bool _permitDeserialization;  // can deserialize BinaryFormatted resources
-        private object? _binaryFormatter; // binary formatter instance to use for deserializing
-
-        // statics used to dynamically call into BinaryFormatter
-        // When successfully located s_binaryFormatterType will point to the BinaryFormatter type
-        // and s_deserializeMethod will point to an unbound delegate to the deserialize method.
-        private static Type s_binaryFormatterType;
-        private static Func<object?, Stream, object> s_deserializeMethod;
-
         // We'll include a separate code path that uses UnmanagedMemoryStream to
         // avoid allocating String objects and the like.
         private UnmanagedMemoryStream? _ums;
@@ -117,7 +113,12 @@ namespace System.Resources
         private int _version;
 
 
-        public ResourceReader(string fileName)
+        public
+#if RESOURCES_EXTENSIONS
+        DeserializingResourceReader(string fileName)
+#else
+        ResourceReader(string fileName)
+#endif
         {
             _resCache = new Dictionary<string, ResourceLocator>(FastResourceComparer.Default);
             _store = new BinaryReader(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, DefaultFileStreamBufferSize, FileOptions.RandomAccess), Encoding.UTF8);
@@ -133,7 +134,12 @@ namespace System.Resources
             }
         }
 
-        public ResourceReader(Stream stream)
+        public
+#if RESOURCES_EXTENSIONS
+        DeserializingResourceReader(Stream stream)
+#else
+        ResourceReader(Stream stream)
+#endif
         {
             if (stream == null)
                 throw new ArgumentNullException(nameof(stream));
@@ -148,27 +154,6 @@ namespace System.Resources
             ReadResources();
         }
 
-        // This is the constructor the RuntimeResourceSet calls,
-        // passing in the stream to read from and the RuntimeResourceSet's 
-        // internal hash table (hash table of names with file offsets
-        // and values, coupled to this ResourceReader).
-        internal ResourceReader(Stream stream, Dictionary<string, ResourceLocator> resCache, bool permitDeserialization)
-        {
-            Debug.Assert(stream != null, "Need a stream!");
-            Debug.Assert(stream.CanRead, "Stream should be readable!");
-            Debug.Assert(resCache != null, "Need a Dictionary!");
-
-            _resCache = resCache;
-            _store = new BinaryReader(stream, Encoding.UTF8);
-
-            _ums = stream as UnmanagedMemoryStream;
-
-            _permitDeserialization = permitDeserialization;
-
-            ReadResources();
-        }
-
-
         public void Close()
         {
             Dispose(true);
@@ -770,59 +755,6 @@ namespace System.Resources
             return DeserializeObject(typeIndex);
         }
 
-        private object DeserializeObject(int typeIndex)
-        {
-            if (!_permitDeserialization)
-            {
-                throw new NotSupportedException(SR.NotSupported_ResourceObjectSerialization);
-            }
-
-            if (_binaryFormatter == null)
-            {
-                InitializeBinaryFormatter();
-            }
-
-            Type type = FindType(typeIndex);
-  
-            object graph = s_deserializeMethod(_binaryFormatter, _store.BaseStream);
-            
-            // guard against corrupted resources
-            if (graph.GetType() != type)
-                throw new BadImageFormatException(SR.Format(SR.BadImageFormat_ResType_SerBlobMismatch, type.FullName, graph.GetType().FullName));
-            return graph;
-        }
-
-        private void InitializeBinaryFormatter()
-        {
-            LazyInitializer.EnsureInitialized(ref s_binaryFormatterType, () =>
-                Type.GetType("System.Runtime.Serialization.Formatters.Binary.BinaryFormatter, System.Runtime.Serialization.Formatters, Version=0.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a",
-                throwOnError: true));
-
-            LazyInitializer.EnsureInitialized(ref s_deserializeMethod, () =>
-               {
-                   MethodInfo binaryFormatterDeserialize = s_binaryFormatterType.GetMethod("Deserialize", new Type[] { typeof(Stream) });
-
-                    // create an unbound delegate that can accept a BinaryFormatter instance as object
-                    return (Func<object?, Stream, object>)typeof(ResourceReader)
-                            .GetMethod(nameof(CreateUntypedDelegate), BindingFlags.NonPublic | BindingFlags.Static)
-                            .MakeGenericMethod(s_binaryFormatterType)
-                            .Invoke(null, new object[] { binaryFormatterDeserialize });
-               });
-
-            _binaryFormatter = Activator.CreateInstance(s_binaryFormatterType);
-        }
-
-        // generic method that we specialize at runtime once we've loaded the BinaryFormatter type
-        // permits creating an unbound delegate so that we can avoid reflection after the initial
-        // lightup code completes.
-        private static Func<object, Stream, object> CreateUntypedDelegate<TInstance>(MethodInfo method)
-        {
-            Func<TInstance, Stream, object> typedDelegate = (Func<TInstance, Stream, object>)Delegate.CreateDelegate(typeof(Func<TInstance, Stream, object>), null, method);
-
-            return (obj, stream) => typedDelegate((TInstance)obj, stream);
-        }
-
         // Reads in the header information for a .resources file.  Verifies some
         // of the assumptions about this resource set, and builds the class table
         // for the default resource file format.
@@ -875,7 +807,7 @@ namespace System.Resources
                 // Note ResourceWriter & InternalResGen use different Strings.
                 string readerType = _store.ReadString();
 
-                if (!ResourceManager.IsDefaultType(readerType, ResourceManager.ResReaderTypeName))
+                if (!ValidateReaderType(readerType))
                     throw new NotSupportedException(SR.Format(SR.NotSupported_WrongResourceReader_Type, readerType));
 
                 // Skip over type name for a suitable ResourceSet
@@ -1044,72 +976,6 @@ namespace System.Resources
             return _typeTable[typeIndex]!; // TODO-NULLABLE: https://github.com/dotnet/roslyn/issues/34644
         }
 
-        public void GetResourceData(string resourceName, out string resourceType, out byte[] resourceData)
-        {
-            if (resourceName == null)
-                throw new ArgumentNullException(nameof(resourceName));
-            if (_resCache == null)
-                throw new InvalidOperationException(SR.ResourceReaderIsClosed);
-
-            // Get the type information from the data section.  Also,
-            // sort all of the data section's indexes to compute length of
-            // the serialized data for this type (making sure to subtract
-            // off the length of the type code).
-            int[] sortedDataPositions = new int[_numResources];
-            int dataPos = FindPosForResource(resourceName);
-            if (dataPos == -1)
-            {
-                throw new ArgumentException(SR.Format(SR.Arg_ResourceNameNotExist, resourceName));
-            }
-
-            lock (this)
-            {
-                // Read all the positions of data within the data section.
-                for (int i = 0; i < _numResources; i++)
-                {
-                    _store.BaseStream.Position = _nameSectionOffset + GetNamePosition(i);
-                    // Skip over name of resource
-                    int numBytesToSkip = _store.Read7BitEncodedInt();
-                    if (numBytesToSkip < 0)
-                    {
-                        throw new FormatException(SR.Format(SR.BadImageFormat_ResourcesNameInvalidOffset, numBytesToSkip));
-                    }
-                    _store.BaseStream.Position += numBytesToSkip;
-
-                    int dPos = _store.ReadInt32();
-                    if (dPos < 0 || dPos >= _store.BaseStream.Length - _dataSectionOffset)
-                    {
-                        throw new FormatException(SR.Format(SR.BadImageFormat_ResourcesDataInvalidOffset, dPos));
-                    }
-                    sortedDataPositions[i] = dPos;
-                }
-                Array.Sort(sortedDataPositions);
-
-                int index = Array.BinarySearch(sortedDataPositions, dataPos);
-                Debug.Assert(index >= 0 && index < _numResources, "Couldn't find data position within sorted data positions array!");
-                long nextData = (index < _numResources - 1) ? sortedDataPositions[index + 1] + _dataSectionOffset : _store.BaseStream.Length;
-                int len = (int)(nextData - (dataPos + _dataSectionOffset));
-                Debug.Assert(len >= 0 && len <= (int)_store.BaseStream.Length - dataPos + _dataSectionOffset, "Length was negative or outside the bounds of the file!");
-
-                // Read type code then byte[]
-                _store.BaseStream.Position = _dataSectionOffset + dataPos;
-                ResourceTypeCode typeCode = (ResourceTypeCode)_store.Read7BitEncodedInt();
-                if (typeCode < 0 || typeCode >= ResourceTypeCode.StartOfUserTypes + _typeTable.Length)
-                {
-                    throw new BadImageFormatException(SR.BadImageFormat_InvalidType);
-                }
-                resourceType = TypeNameFromTypeCode(typeCode);
-
-                // The length must be adjusted to subtract off the number 
-                // of bytes in the 7 bit encoded type code.
-                len -= (int)(_store.BaseStream.Position - (_dataSectionOffset + dataPos));
-                byte[] bytes = _store.ReadBytes(len);
-                if (bytes.Length != len)
-                    throw new FormatException(SR.BadImageFormat_ResourceNameCorrupted);
-                resourceData = bytes;
-            }
-        }
-
         private string TypeNameFromTypeCode(ResourceTypeCode typeCode)
         {
             Debug.Assert(typeCode >= 0, "can't be negative");
index c2d27d2..dbd74b2 100644 (file)
 ** 
 ===========================================================*/
 
-using System;
-using System.IO;
 using System.Collections;
 using System.Collections.Generic;
-using System.Globalization;
-using System.Reflection;
-using System.Runtime.Versioning;
 using System.Diagnostics;
 
 namespace System.Resources
+#if RESOURCES_EXTENSIONS
+    .Extensions
+#endif
 {
+#if RESOURCES_EXTENSIONS
+    using ResourceReader = DeserializingResourceReader;
+#endif
     // A RuntimeResourceSet stores all the resources defined in one 
     // particular CultureInfo, with some loading optimizations.
     //
@@ -192,6 +193,7 @@ namespace System.Resources
         // the resources once, adding them into the table.
         private bool _haveReadFromReader;
 
+#if !RESOURCES_EXTENSIONS
         internal RuntimeResourceSet(string fileName) : base(false)
         {
             _resCache = new Dictionary<string, ResourceLocator>(FastResourceComparer.Default);
@@ -206,6 +208,26 @@ namespace System.Resources
             _defaultReader = new ResourceReader(stream, _resCache, permitDeserialization);
             Reader = _defaultReader;
         }
+#else
+        private IResourceReader Reader => _defaultReader!;
+
+        internal RuntimeResourceSet(IResourceReader reader) :
+            // explicitly do not call IResourceReader constructor since it caches all resources
+            // the purpose of RuntimeResourceSet is to lazily load and cache.
+            base()
+        {
+            if (reader == null)
+                throw new ArgumentNullException(nameof(reader));
+
+            _defaultReader = reader as DeserializingResourceReader ?? throw new ArgumentException(SR.Format(SR.NotSupported_WrongResourceReader_Type, reader.GetType()), nameof(reader));
+            _resCache = new Dictionary<string, ResourceLocator>(FastResourceComparer.Default);
+            
+            // in the CoreLib version RuntimeResourceSet creates ResourceReader and passes this in, 
+            // in the custom case ManifestBasedResourceReader creates the ResourceReader and passes it in
+            // so we must initialize the cache here.
+            _defaultReader._resCache = _resCache;
+        }
+#endif
 
         protected override void Dispose(bool disposing)
         {