[release/8.0] [mono][metadata] Replace use of mem manager lock with loader lock ...

* [mono][metadata] Replace use of mem manager lock with loader lock

Hash table operations under the mem manager lock could end up taking the loader lock when performing type comparison, in the case where custom modifiers needed to be loaded. Use the loader lock instead to prevent deadlocks.

* [mono][metadata] Use loader lock during generic class hash table lookup

---------

Co-authored-by: Vlad Brezae <brezaevlad@gmail.com>

diff --git a/src/mono/mono/metadata/metadata.c b/src/mono/mono/metadata/metadata.c
index 7c7cd44..f9abe65 100644 (file)
--- a/src/mono/mono/metadata/metadata.c
+++ b/src/mono/mono/metadata/metadata.c
@@ -3434,7 +3434,8 @@ mono_metadata_get_canonical_generic_inst (MonoGenericInst *candidate)
        MonoMemoryManager *mm = mono_mem_manager_get_generic (data.images, data.nimages);
        collect_data_free (&data);
 
-       mono_mem_manager_lock (mm);
+       // Hashtable key equal func can take loader lock
+       mono_loader_lock ();
 
        if (!mm->ginst_cache)
                mm->ginst_cache = g_hash_table_new_full (mono_metadata_generic_inst_hash, mono_metadata_generic_inst_equal, NULL, (GDestroyNotify)free_generic_inst);
@@ -3456,7 +3457,7 @@ mono_metadata_get_canonical_generic_inst (MonoGenericInst *candidate)
                g_hash_table_insert (mm->ginst_cache, ginst, ginst);
        }
 
-       mono_mem_manager_unlock (mm);
+       mono_loader_unlock ();
 
        return ginst;
 }
@@ -3467,7 +3468,8 @@ mono_metadata_get_canonical_aggregate_modifiers (MonoAggregateModContainer *cand
        g_assert (candidate->count > 0);
        MonoMemoryManager *mm = mono_metadata_get_mem_manager_for_aggregate_modifiers (candidate);
 
-       mono_mem_manager_lock (mm);
+       // Hashtable key equal func can take loader lock
+       mono_loader_lock ();
 
        if (!mm->aggregate_modifiers_cache)
                mm->aggregate_modifiers_cache = g_hash_table_new_full (aggregate_modifiers_hash, aggregate_modifiers_equal, NULL, (GDestroyNotify)free_aggregate_modifiers);
@@ -3484,7 +3486,7 @@ mono_metadata_get_canonical_aggregate_modifiers (MonoAggregateModContainer *cand
 
                g_hash_table_insert (mm->aggregate_modifiers_cache, amods, amods);
        }
-       mono_mem_manager_unlock (mm);
+       mono_loader_unlock ();
        return amods;
 }
 
@@ -3543,7 +3545,8 @@ mono_metadata_lookup_generic_class (MonoClass *container_class, MonoGenericInst
        if (gclass)
                return gclass;
 
-       mono_mem_manager_lock (mm);
+       // Hashtable key equal func can take loader lock
+       mono_loader_lock ();
 
        gclass = mono_mem_manager_alloc0 (mm, sizeof (MonoGenericClass));
        if (is_dynamic)
@@ -3563,7 +3566,7 @@ mono_metadata_lookup_generic_class (MonoClass *container_class, MonoGenericInst
 
        // g_hash_table_insert (set->gclass_cache, gclass, gclass);
 
-       mono_mem_manager_unlock (mm);
+       mono_loader_unlock ();
 
        return gclass2;
 }