From: Charles Giessen <charles@lunarg.com>
Date: Thu, 8 Dec 2022 00:15:22 +0000 (-0700)
Subject: Try to catch OOM errors when loading binaries
X-Git-Tag: upstream/1.3.240~12
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=366966e85b85fdfce8e365360cd37bb28bb9a642;p=platform%2Fupstream%2FVulkan-Loader.git

Try to catch OOM errors when loading binaries

If there is an error loading a dynamic library, check to see if it is
the result of an OOM condition. Then return the appropriate OOM error
code up the callchain. This commit is the result of an OOM condition
when trying to load the validation layers, which resulted in the return
code VK_ERROR_LAYER_NOT_PRESENT. This is very misleading. If the
loader can catch then report more accurate information, then it should.
---

diff --git a/loader/loader.c b/loader/loader.c
index fff5ca9b..bf51d3c2 100644
--- a/loader/loader.c
+++ b/loader/loader.c
@@ -170,6 +170,14 @@ void loader_handle_load_library_error(const struct loader_instance *inst, const
         if (NULL != lib_status) {
             *lib_status = LOADER_LAYER_LIB_ERROR_WRONG_BIT_TYPE;
         }
+    }
+    // Check if the error is due to lack of memory
+    // "with error 8" is the windows error code for OOM cases, aka ERROR_NOT_ENOUGH_MEMORY
+    // Linux doesn't have such a nice error message - only if there are reported issues should this be called
+    else if (strstr(error_message, " with error 8") != NULL) {
+        if (NULL != lib_status) {
+            *lib_status = LOADER_LAYER_LIB_ERROR_OUT_OF_MEMORY;
+        }
     } else if (NULL != lib_status) {
         *lib_status = LOADER_LAYER_LIB_ERROR_FAILED_TO_LOAD;
     }
@@ -1373,7 +1381,11 @@ static VkResult loader_scanned_icd_add(const struct loader_instance *inst, struc
 #endif
     if (NULL == handle) {
         loader_handle_load_library_error(inst, filename, lib_status);
-        res = VK_ERROR_INCOMPATIBLE_DRIVER;
+        if (lib_status && *lib_status == LOADER_LAYER_LIB_ERROR_OUT_OF_MEMORY) {
+            res = VK_ERROR_OUT_OF_HOST_MEMORY;
+        } else {
+            res = VK_ERROR_INCOMPATIBLE_DRIVER;
+        }
         goto out;
     }
 
@@ -3551,6 +3563,7 @@ VkResult loader_icd_scan(const struct loader_instance *inst, struct loader_icd_t
                     break;
                 }
                 case LOADER_LAYER_LIB_SUCCESS_LOADED:
+                case LOADER_LAYER_LIB_ERROR_OUT_OF_MEMORY:
                     // Shouldn't be able to reach this but if it is, best to report a debug
                     loader_log(inst, VULKAN_LOADER_WARN_BIT | VULKAN_LOADER_DRIVER_BIT, 0,
                                "Shouldn't reach this. A valid version of requested ICD %s was loaded but something bad "
@@ -4450,6 +4463,9 @@ VkResult loader_create_instance_chain(const VkInstanceCreateInfo *pCreateInfo, c
             }
 
             lib_handle = loader_open_layer_file(inst, layer_prop);
+            if (layer_prop->lib_status == LOADER_LAYER_LIB_ERROR_OUT_OF_MEMORY) {
+                return VK_ERROR_OUT_OF_HOST_MEMORY;
+            }
             if (!lib_handle) {
                 continue;
             }
@@ -4599,6 +4615,7 @@ VkResult loader_create_instance_chain(const VkInstanceCreateInfo *pCreateInfo, c
                                ending);
                     break;
                 case LOADER_LAYER_LIB_SUCCESS_LOADED:
+                case LOADER_LAYER_LIB_ERROR_OUT_OF_MEMORY:
                     // Shouldn't be able to reach this but if it is, best to report a debug
                     loader_log(inst, log_flag, 0,
                                "Shouldn't reach this. A valid version of requested layer %s was loaded but was not found in the "
diff --git a/loader/loader_common.h b/loader/loader_common.h
index d79cca70..5e5adb22 100644
--- a/loader/loader_common.h
+++ b/loader/loader_common.h
@@ -113,6 +113,7 @@ enum loader_layer_library_status {
 
     LOADER_LAYER_LIB_ERROR_WRONG_BIT_TYPE = 20,
     LOADER_LAYER_LIB_ERROR_FAILED_TO_LOAD = 21,
+    LOADER_LAYER_LIB_ERROR_OUT_OF_MEMORY = 22,
 };
 
 enum layer_type_flags {