Use QueryAdapter APIs as per MS directives
authorNikhil Joshi <nikhilj@nvidia.com>
Tue, 21 May 2019 10:28:18 +0000 (15:58 +0530)
committerNikhil Joshi <nikhilj@nvidia.com>
Tue, 21 May 2019 11:48:58 +0000 (17:18 +0530)
Update ICD loader to use QueryAdapter APIs
as per MS directives for paravirtualization.

Merge branch 'skudchad-ocl-wddm24'
of https://github.com/BKlamik-MSFT/OpenCL-ICD-Loader

13 files changed:
CMakeLists.txt
README.md
build_using_cmake.bat
loader/icd.h
loader/icd_dispatch.c
loader/linux/icd_linux.c
loader/windows/OpenCL.rc
loader/windows/icd_windows.c
loader/windows/icd_windows_dxgk.c
loader/windows/icd_windows_hkr.c
loader/windows/icd_windows_hkr.h
test/loader_test/test_buffer_object.c
test/loader_test/test_kernel.c

index c750916..d25713c 100644 (file)
@@ -41,6 +41,7 @@ if (WIN32)
     list (APPEND OPENCL_ICD_LOADER_SOURCES 
         loader/windows/icd_windows.c
         loader/windows/icd_windows_hkr.c
+        loader/windows/icd_windows_dxgk.c 
         loader/windows/OpenCL.def
         loader/windows/OpenCL.rc)
     # Only add the DXSDK include directory if the environment variable is
@@ -48,6 +49,7 @@ if (WIN32)
     # only required in rare cases.
     if (DEFINED ENV{DXSDK_DIR})
         include_directories ($ENV{DXSDK_DIR}/Include)
+        include_directories ($ENV{WDK}/km)
     endif ()
 else ()
     list (APPEND OPENCL_ICD_LOADER_SOURCES
index 8d22438..6b41b33 100644 (file)
--- a/README.md
+++ b/README.md
@@ -3,13 +3,26 @@
 The build system will build ICD Loader library (OpenCL.dll or libOpenCL.so), the
 ICD Loader Test binary (icd_loader_test), and some helper libraries for the test.
 
+1. See inc/readme.txt about downloading or symlink the OpenCL headers.
+    Ex: https://github.com/KhronosGroup/OpenCL-Headers
+
 ## Linux
 
 Run "make"
 
 ## Windows
 
-Run "build_using_cmake.bat"
+1. Install recent Windows WDK, for access to d3dkmthk.h
+    Currently at https://docs.microsoft.com/en-us/windows-hardware/drivers/download-the-wdk
+
+2. Establish environment variable WDK to include directory
+    Ex: set WDK=C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0
+
+2. Modify environment for your compiler.
+    Ex: Command line builds should modify "build_using_cmake.bat" 
+    Ex: Visual Studio CMake integration should modify the "buildRoot" to match the BUILD_DIR in "build_using_cmake.bat"
+
+3. Build either using the command line to run "build_using_cmake.bat" or with Visual Studio CMake integration
 
 ## Running ICD Test
 
index f0044db..727a55d 100644 (file)
@@ -1,3 +1,11 @@
+@set WDK=C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0
+@if EXIST "%WDK%\km\d3dkmthk.h" goto found_WDK
+  @echo Windows WDK not found at "%WDK%".
+  @echo Download it from https://docs.microsoft.com/en-us/windows-hardware/drivers/download-the-wdk
+  @echo Then, fix build_using_cmake.bat
+  @goto :eof
+:found_WDK
+
 call "%VS90COMNTOOLS%/vsvars32.bat"
 
 set BUILD_DIR=build
index a1b6969..fdc2a53 100644 (file)
@@ -121,6 +121,11 @@ void khrIcdContextPropertiesGetPlatform(
     const cl_context_properties *properties, 
     cl_platform_id *outPlatform);
 
+// translate device type to platform
+void khrIcdDeviceTypeGetPlatform(
+    cl_device_type device_type,
+    cl_platform_id *outPlatform);
+
 // internal tracing macros
 #if 0
     #include <stdio.h>
index df967cb..0bd7b04 100644 (file)
@@ -99,6 +99,17 @@ clGetDeviceIDs(cl_platform_id   platform,
 {
     // initialize the platforms (in case they have not been already)
     khrIcdInitialize();
+
+    // determine the platform to use from the device_type specified
+    if (!platform)
+    {
+        khrIcdDeviceTypeGetPlatform(device_type, &platform);
+    }
+    if (!platform && khrIcdVendors != NULL)
+    {
+        platform = khrIcdVendors[0].platform;
+    }
+
     KHR_ICD_VALIDATE_HANDLE_RETURN_ERROR(platform, CL_INVALID_PLATFORM);   
     return platform->dispatch->clGetDeviceIDs(
         platform,
@@ -196,8 +207,16 @@ clCreateContextFromType(const cl_context_properties * properties,
     // initialize the platforms (in case they have not been already)
     khrIcdInitialize();
 
-    // determine the platform to use from the properties specified
+    // determine the platform to use from the properties and device_type specified
     khrIcdContextPropertiesGetPlatform(properties, &platform);
+    if (!platform)
+    {
+        khrIcdDeviceTypeGetPlatform(device_type, &platform);
+    }
+    if (!platform && khrIcdVendors != NULL)
+    {
+        platform = khrIcdVendors[0].platform;
+    }
 
     // validate the platform handle and dispatch
     KHR_ICD_VALIDATE_HANDLE_RETURN_HANDLE(platform, CL_INVALID_PLATFORM);
@@ -1823,6 +1842,15 @@ CL_API_ENTRY cl_int CL_API_CALL clGetGLContextInfoKHR(
 
     // determine the platform to use from the properties specified
     khrIcdContextPropertiesGetPlatform(properties, &platform);
+    // determine the platform to use from the device_type specified
+    if (!platform)
+    {
+        khrIcdDeviceTypeGetPlatform(device_type, &platform);
+    }
+    if (!platform && khrIcdVendors != NULL)
+    {
+        platform = khrIcdVendors[0].platform;
+    }
 
     KHR_ICD_VALIDATE_HANDLE_RETURN_ERROR(platform, CL_INVALID_PLATFORM);    
     return platform->dispatch->clGetGLContextInfoKHR(
index f6bb7b6..51b3911 100644 (file)
@@ -167,3 +167,7 @@ void khrIcdOsLibraryUnload(void *library)
     dlclose(library);
 }
 
+// implement device type platform behavior
+void khrIcdDeviceTypeGetPlatform(cl_device_type device_type, cl_platform_id *outPlatform)
+{
+}
index 561e6ba..532d5ea 100644 (file)
@@ -21,8 +21,8 @@
 #ifdef RC_INVOKED
 
 VS_VERSION_INFO VERSIONINFO
-FILEVERSION    2,2,1,0
-PRODUCTVERSION 2,2,1,0
+FILEVERSION    2,2,2,0
+PRODUCTVERSION 2,2,2,0
 FILETYPE       VFT_DLL
 
 BEGIN
@@ -33,7 +33,7 @@ BEGIN
             VALUE "FileDescription" ,"OpenCL Client DLL"
             VALUE "ProductName"     ,"Khronos OpenCL ICD"
             VALUE "LegalCopyright"  ,"Copyright \251 The Khronos Group Inc 2016-2019"
-            VALUE "FileVersion"     ,"2.2.1.0"
+            VALUE "FileVersion"     ,"2.2.2.0"
             VALUE "CompanyName"     ,"Khronos Group"
             VALUE "InternalName"    ,"OpenCL"
             VALUE "OriginalFilename","OpenCL.dll"
index ec86184..da0888c 100644 (file)
  */
 
 #include "icd.h"
+#include <windows.h>
 #include "icd_windows_hkr.h"
 #include "icd_windows_dxgk.h"
 #include <stdio.h>
-#include <windows.h>
 #include <winreg.h>
 
+#include <initguid.h>
+#include <dxgi.h>
+typedef HRESULT (WINAPI *PFN_CREATE_DXGI_FACTORY)(REFIID, void **);
+
 static INIT_ONCE initialized = INIT_ONCE_STATIC_INIT;
 
+typedef struct WinAdapter
+{
+    char * szName;
+    LUID luid;
+} WinAdapter;
+
+LUID ZeroLuid = { 0, 0 };
+
+static WinAdapter* pWinAdapterBegin = NULL;
+static WinAdapter* pWinAdapterEnd = NULL;
+static WinAdapter* pWinAdapterCapacity = NULL;
+
+void AdapterAdd(const char* szName, LUID luid)
+{
+    if (pWinAdapterEnd == pWinAdapterCapacity)
+    {
+        size_t OldCapacity = pWinAdapterCapacity - pWinAdapterBegin;
+        size_t NewCapacity = OldCapacity;
+        if (0 == NewCapacity)
+        {
+            NewCapacity = 1;
+        }
+        NewCapacity *= 2;
+
+        WinAdapter* pNewBegin = malloc(NewCapacity * sizeof(*pWinAdapterBegin));
+        if (pNewBegin)
+        {
+            if (pWinAdapterBegin)
+            {
+                memcpy(pNewBegin, pWinAdapterBegin, OldCapacity * sizeof(*pWinAdapterBegin));
+                free(pWinAdapterBegin);
+            }
+            pWinAdapterCapacity = pNewBegin + NewCapacity;
+            pWinAdapterEnd = pNewBegin + OldCapacity;
+            pWinAdapterBegin = pNewBegin;
+        }
+    }
+    if (pWinAdapterEnd != pWinAdapterCapacity)
+    {
+        size_t nameLen = strlen(szName) + 1;
+        if (pWinAdapterEnd->szName = malloc(nameLen))
+        {
+            memcpy(pWinAdapterEnd->szName, szName, nameLen * sizeof(*szName));
+            pWinAdapterEnd->luid = luid;
+            ++pWinAdapterEnd;
+        }
+    }
+}
+
 /*
  * 
  * Vendor enumeration functions
@@ -103,7 +156,46 @@ BOOL CALLBACK khrIcdOsVendorsEnumerate(PINIT_ONCE InitOnce, PVOID Parameter, PVO
         }
 
         // add the library
-        khrIcdVendorAdd(cszLibraryName);
+        AdapterAdd(cszLibraryName, ZeroLuid);
+    }
+
+    // Add adapters according to DXGI's preference order
+    HMODULE hDXGI = LoadLibrary("dxgi.dll");
+    if (hDXGI)
+    {
+        IDXGIFactory* pFactory = NULL;
+        PFN_CREATE_DXGI_FACTORY pCreateDXGIFactory = (PFN_CREATE_DXGI_FACTORY)GetProcAddress(hDXGI, "CreateDXGIFactory");
+        HRESULT hr = pCreateDXGIFactory(&IID_IDXGIFactory, &pFactory);
+        if (SUCCEEDED(hr))
+        {
+            UINT i = 0;
+            IDXGIAdapter* pAdapter = NULL;
+            while (SUCCEEDED(pFactory->lpVtbl->EnumAdapters(pFactory, i++, &pAdapter)))
+            {
+                DXGI_ADAPTER_DESC AdapterDesc;
+                pAdapter->lpVtbl->GetDesc(pAdapter, &AdapterDesc);
+
+                for (WinAdapter* iterAdapter = pWinAdapterBegin; iterAdapter != pWinAdapterEnd; ++iterAdapter)
+                {
+                    if (iterAdapter->luid.LowPart == AdapterDesc.AdapterLuid.LowPart
+                        && iterAdapter->luid.HighPart == AdapterDesc.AdapterLuid.HighPart)
+                    {
+                        khrIcdVendorAdd(iterAdapter->szName);
+                        break;
+                    }
+                }
+
+                pAdapter->lpVtbl->Release(pAdapter);
+            }
+            pFactory->lpVtbl->Release(pFactory);
+        }
+        FreeLibrary(hDXGI);
+    }
+
+    // Go through the list again, putting any remaining adapters at the end of the list in an undefined order
+    for (WinAdapter* iterAdapter = pWinAdapterBegin; iterAdapter != pWinAdapterEnd; ++iterAdapter)
+    {
+        khrIcdVendorAdd(iterAdapter->szName);
     }
 
     result = RegCloseKey(platformsKey);
@@ -149,3 +241,9 @@ void khrIcdOsLibraryUnload(void *library)
     FreeLibrary( (HMODULE)library);
 }
 
+// implement device type platform behavior
+void khrIcdDeviceTypeGetPlatform(cl_device_type device_type, cl_platform_id *outPlatform)
+{
+    // CL_DEVICE_TYPE_GPU => Should likely check Direct3D user-mode driver caps
+    // CL_DEVICE_TYPE_CPU => Should likely check D3DKMT_ADAPTERTYPE::SoftwareDevice
+}
index 1e46f6b..c5e3a5a 100644 (file)
@@ -37,6 +37,7 @@
 
 #include "icd.h"
 #include <windows.h>
+#include "icd_windows_hkr.h"
 #include "icd_windows_dxgk.h"
 #include <assert.h>
 
@@ -52,14 +53,14 @@ typedef LONG NTSTATUS;
 bool khrIcdOsVendorsEnumerateDXGK(void)
 {
 #if defined(DXGKDDI_INTERFACE_VERSION_WDDM2_4) && (DXGKDDI_INTERFACE_VERSION >= DXGKDDI_INTERFACE_VERSION_WDDM2_4)
+    // Get handle to GDI Runtime
+    HMODULE h = LoadLibrary("gdi32.dll");
+    if (h && GetProcAddress((HMODULE)h, "D3DKMTSubmitPresentBltToHwQueue")) // OS Version check
     {
         D3DKMT_ADAPTERINFO* pAdapterInfo;
         D3DKMT_ENUMADAPTERS2 EnumAdapters;
         NTSTATUS Status = STATUS_SUCCESS;
 
-        // Get handle to GDI Runtime
-        HMODULE h = LoadLibrary("gdi32.dll");
-
         char cszLibraryName[1024] = { 0 };
         EnumAdapters.NumAdapters = 0;
         EnumAdapters.pAdapters = NULL;
@@ -84,6 +85,8 @@ bool khrIcdOsVendorsEnumerateDXGK(void)
             if (pAdapterInfo) free(pAdapterInfo);
             return FALSE;
         }
+        const char* cszOpenCLRegKeyName = GetOpenCLRegKeyName();
+        const int OpenCLRegKeyNameSize = (int)(strlen(cszOpenCLRegKeyName) + 1);
         for (UINT AdapterIndex = 0; AdapterIndex < EnumAdapters.NumAdapters; AdapterIndex++)
         {
             D3DDDI_QUERYREGISTRY_INFO QueryArgs = {0};
@@ -91,12 +94,19 @@ bool khrIcdOsVendorsEnumerateDXGK(void)
             D3DDDI_QUERYREGISTRY_INFO* pQueryBuffer = NULL;
             QueryArgs.QueryType = D3DDDI_QUERYREGISTRY_ADAPTERKEY;
             QueryArgs.QueryFlags.TranslatePath = TRUE;
-            QueryArgs.ValueType = REG_MULTI_SZ;
 #ifdef _WIN64
             wcscpy_s(QueryArgs.ValueName, ARRAYSIZE(L"OpenCLDriverName"), L"OpenCLDriverName");
 #else
             wcscpy_s(QueryArgs.ValueName, ARRAYSIZE(L"OpenCLDriverNameWow"), L"OpenCLDriverNameWow");
 #endif
+            QueryArgs.ValueType = REG_SZ;
+            MultiByteToWideChar(
+                CP_ACP,
+                0,
+                cszOpenCLRegKeyName,
+                OpenCLRegKeyNameSize,
+                QueryArgs.ValueName,
+                ARRAYSIZE(QueryArgs.ValueName));
             D3DKMT_QUERYADAPTERINFO QueryAdapterInfo = {0};
             QueryAdapterInfo.hAdapter = pAdapterInfo[AdapterIndex].hAdapter;
             QueryAdapterInfo.Type = KMTQAITYPE_QUERYREGISTRY;
@@ -105,9 +115,9 @@ bool khrIcdOsVendorsEnumerateDXGK(void)
             Status = D3DKMTQueryAdapterInfo(&QueryAdapterInfo);
             if (!NT_SUCCESS(Status))
             {
-                FreeLibrary(h);
-                if (pAdapterInfo) free(pAdapterInfo);
-                return FALSE;
+                // Continue trying to get as much info on each adapter as possible.
+                // It's too late to return FALSE and claim WDDM2_4 enumeration is not available here.
+                continue;
             }
             if (NT_SUCCESS(Status) && pQueryArgs->Status == D3DDDI_QUERYREGISTRY_STATUS_BUFFER_OVERFLOW)
             {
@@ -139,7 +149,7 @@ bool khrIcdOsVendorsEnumerateDXGK(void)
                         i++;
                         pWchar++;
                     }
-                    if (i < 1023) khrIcdVendorAdd(cszLibraryName);
+                    if (i < 1023) AdapterAdd(cszLibraryName, pAdapterInfo[AdapterIndex].AdapterLuid);
                 }
             }
             if (pQueryBuffer) free(pQueryBuffer);
index 092746a..5a02704 100644 (file)
@@ -17,8 +17,9 @@
  */
 
 #include "icd.h"
-#include "icd_windows_hkr.h"
 #include <windows.h>
+#include "icd_windows_hkr.h"
+#include "icd_windows_dxgk.h"
 #include <cfgmgr32.h>
 #include <assert.h>
 #include <stdbool.h>
@@ -51,7 +52,7 @@ static const char OPENCL_REG_SUB_KEY_WOW[] = "OpenCLDriverNameWow";
 #endif
 
 // Do not free the memory returned by this function.
-static const char* GetOpenCLRegKeyName(void)
+const char* GetOpenCLRegKeyName(void)
 {
 #ifdef _WIN64
     return OPENCL_REG_SUB_KEY;
@@ -136,7 +137,7 @@ static bool ReadOpenCLKey(DEVINST dnDevNode)
 
         KHR_ICD_TRACE("    Path: %s\n", cszOclPath);
 
-        khrIcdVendorAdd(cszOclPath);
+        AdapterAdd(cszOclPath, ZeroLuid);
 
         bRet = true;
     }
index 698fe5a..e8f777e 100644 (file)
 #include <stdbool.h>
 
 bool khrIcdOsVendorsEnumerateHKR(void);
+
+LUID ZeroLuid;
+
+void AdapterAdd(const char* szName, LUID luid);
+
+// Do not free the memory returned by this function.
+const char* GetOpenCLRegKeyName(void);
index 1710e88..1de6ddb 100644 (file)
@@ -1,5 +1,4 @@
 #include <stdlib.h>
-
 #include <CL/cl.h>
 #include "param_struct.h"
 #include <platform/icd_test_log.h>
index a53e980..ab63932 100644 (file)
@@ -154,7 +154,7 @@ int test_clEnqueueMigrateMemObjects(const struct clEnqueueMigrateMemObjects_st*
                      &event);
 
     ret_val=clEnqueueMigrateMemObjects(command_queue,
-            data->num_mem_objects, 
+            (cl_uint)data->num_mem_objects, 
             data->mem_objects,
             data->flags,
             data->num_events_in_wait_list,