[CMAKE] Improve FindLLVM to handle llvm-prefix with space. (#6466)
authorTianqi Chen <tqchen@users.noreply.github.com>
Sun, 13 Sep 2020 16:58:49 +0000 (09:58 -0700)
committerGitHub <noreply@github.com>
Sun, 13 Sep 2020 16:58:49 +0000 (09:58 -0700)
* [CMAKE] Improve FindLLVM to handle llvm-prefix with space.

Useful for windows settings where llvm can sits in Program Files.
Also updated the windows compiler to use clang.

* Additional updates

apps/howto_deploy/prepare_test_libs.py
cmake/util/FindLLVM.cmake
python/tvm/contrib/cc.py
src/target/llvm/llvm_common.h
tests/python/unittest/test_runtime_rpc.py

index b040be3..a6c7688 100644 (file)
@@ -54,5 +54,5 @@ def prepare_graph_lib(base_path):
 
 if __name__ == "__main__":
     curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-    prepare_test_libs(os.path.join(curr_path, "./lib"))
-    prepare_graph_lib(os.path.join(curr_path, "./lib"))
+    prepare_test_libs(os.path.join(curr_path, "lib"))
+    prepare_graph_lib(os.path.join(curr_path, "lib"))
index 25937e2..d837880 100644 (file)
@@ -61,51 +61,94 @@ macro(find_llvm use_llvm)
     separate_arguments(LLVM_CONFIG)
     execute_process(COMMAND ${LLVM_CONFIG} --libfiles
       RESULT_VARIABLE __llvm_exit_code
-      OUTPUT_VARIABLE __llvm_libfiles)
+      OUTPUT_VARIABLE __llvm_libfiles_space
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
     if(NOT "${__llvm_exit_code}" STREQUAL "0")
       message(FATAL_ERROR "Fatal error executing: ${use_llvm} --libfiles")
     endif()
     execute_process(COMMAND ${LLVM_CONFIG} --system-libs
       RESULT_VARIABLE __llvm_exit_code
-      OUTPUT_VARIABLE __llvm_system_libs)
+      OUTPUT_VARIABLE __llvm_system_libs
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
     if(NOT "${__llvm_exit_code}" STREQUAL "0")
       message(FATAL_ERROR "Fatal error executing: ${use_llvm} --system-libs")
     endif()
     execute_process(COMMAND ${LLVM_CONFIG} --cxxflags
       RESULT_VARIABLE __llvm_exit_code
-      OUTPUT_VARIABLE __llvm_cxxflags)
+      OUTPUT_VARIABLE __llvm_cxxflags_space
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
     if(NOT "${__llvm_exit_code}" STREQUAL "0")
       message(FATAL_ERROR "Fatal error executing: ${use_llvm} --cxxflags")
     endif()
     execute_process(COMMAND ${LLVM_CONFIG} --version
       RESULT_VARIABLE __llvm_exit_code
-      OUTPUT_VARIABLE __llvm_version)
+      OUTPUT_VARIABLE __llvm_version
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
     if(NOT "${__llvm_exit_code}" STREQUAL "0")
       message(FATAL_ERROR "Fatal error executing: ${use_llvm} --version")
     endif()
+    execute_process(COMMAND ${LLVM_CONFIG} --prefix
+      RESULT_VARIABLE __llvm_exit_code
+      OUTPUT_VARIABLE __llvm_prefix
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(NOT "${__llvm_exit_code}" STREQUAL "0")
+      message(FATAL_ERROR "Fatal error executing: ${use_llvm} --prefix")
+    endif()
+    execute_process(COMMAND ${LLVM_CONFIG} --libdir
+      RESULT_VARIABLE __llvm_exit_code
+      OUTPUT_VARIABLE __llvm_libdir
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(NOT "${__llvm_exit_code}" STREQUAL "0")
+      message(FATAL_ERROR "Fatal error executing: ${use_llvm} --libdir")
+    endif()
+    # map prefix => $
+    # to handle the case when the prefix contains space.
+    string(REPLACE ${__llvm_prefix} "$" __llvm_cxxflags ${__llvm_cxxflags_space})
+    string(REPLACE ${__llvm_prefix} "$" __llvm_libfiles ${__llvm_libfiles_space})
     # llvm version
     set(TVM_INFO_LLVM_VERSION ${__llvm_version})
     string(REGEX REPLACE "^([^.]+)\.([^.])+\.[^.]+.*$" "\\1\\2" TVM_LLVM_VERSION ${__llvm_version})
+    string(STRIP ${TVM_LLVM_VERSION} TVM_LLVM_VERSION)
     # definitions
-    string(REGEX MATCHALL "(^| )-D[A-Za-z0-9_]*" LLVM_DEFINITIONS ${__llvm_cxxflags})
+    string(REGEX MATCHALL "(^| )-D[A-Za-z0-9_]*" __llvm_defs ${__llvm_cxxflags})
+    set(LLVM_DEFINTIIONS "")
+    foreach(__flag IN ITEMS ${__llvm_defs})
+      string(STRIP "${__flag}" __llvm_def)
+      list(APPEND LLVM_DEFINITIONS "${__llvm_def}")
+    endforeach()
     # include dir
     string(REGEX MATCHALL "(^| )-I[^ ]*" __llvm_include_flags ${__llvm_cxxflags})
     set(LLVM_INCLUDE_DIRS "")
     foreach(__flag IN ITEMS ${__llvm_include_flags})
       string(REGEX REPLACE "(^| )-I" "" __dir "${__flag}")
-      list(APPEND LLVM_INCLUDE_DIRS "${__dir}")
+      # map $ => prefix
+      string(REPLACE "$" ${__llvm_prefix} __dir_with_prefix "${__dir}")
+      list(APPEND LLVM_INCLUDE_DIRS "${__dir_with_prefix}")
     endforeach()
-    message(STATUS ${LLVM_INCLUDE_DIRS})
     # libfiles
-    string(STRIP ${__llvm_libfiles} __llvm_libfiles)
-    string(STRIP ${__llvm_system_libs} __llvm_system_libs)
-    set(LLVM_LIBS "${__llvm_libfiles} ${__llvm_system_libs}")
-    separate_arguments(LLVM_LIBS)
-    string(STRIP ${TVM_LLVM_VERSION} TVM_LLVM_VERSION)
+    set(LLVM_LIBS "")
+    separate_arguments(__llvm_libfiles)
+    foreach(__flag IN ITEMS ${__llvm_libfiles})
+      # map $ => prefix
+      string(REPLACE "$" ${__llvm_prefix} __lib_with_prefix "${__flag}")
+      list(APPEND LLVM_LIBS "${__lib_with_prefix}")
+    endforeach()
+    separate_arguments(__llvm_system_libs)
+    foreach(__flag IN ITEMS ${__llvm_system_libs})
+      # If the library file ends in .lib try to
+      # also search the llvm_libdir
+      if(__flag MATCHES ".lib$")
+        if(EXISTS "${__llvm_libdir}/${__flag}")
+          set(__flag "${__llvm_libdir}/${__flag}")
+        endif()
+      endif()
+      list(APPEND LLVM_LIBS "${__flag}")
+    endforeach()
   endif()
   if(NOT LLVM_CONFIG STREQUAL "OFF")
-    message(STATUS "Found LLVM_INCLUDE_DIRS=" ${LLVM_INCLUDE_DIRS})
-    message(STATUS "Found LLVM_DEFINITIONS=" ${LLVM_DEFINITIONS})
+    message(STATUS "Found LLVM_INCLUDE_DIRS=" "${LLVM_INCLUDE_DIRS}")
+    message(STATUS "Found LLVM_DEFINITIONS=" "${LLVM_DEFINITIONS}")
+    message(STATUS "Found LLVM_LIBS=" "${LLVM_LIBS}")
     message(STATUS "Found TVM_LLVM_VERSION=" ${TVM_LLVM_VERSION})
     if (${TVM_LLVM_VERSION} LESS 40)
       message(FATAL_ERROR "TVM requires LLVM 4.0 or higher.")
index 1de56d2..ecefe56 100644 (file)
 # under the License.
 """Util to invoke C/C++ compilers in the system."""
 # pylint: disable=invalid-name
-from __future__ import absolute_import as _abs
 import sys
 import subprocess
-import os
 
 from .._ffi.base import py_str
-from .util import tempdir
 
 
 def create_shared(output, objects, options=None, cc="g++"):
@@ -207,66 +204,31 @@ def _linux_compile(output, objects, options, compile_cmd="g++", compile_shared=F
         msg += "\nCommand line: " + " ".join(cmd)
         raise RuntimeError(msg)
 
-
 def _windows_shared(output, objects, options):
-    cl_cmd = ["cl"]
-    cl_cmd += ["-c"]
+    cmd = ["clang"]
+    cmd += ["-O2", "-flto=full", "-fuse-ld=lld-link"]
+
+    if output.endswith(".so") or output.endswith(".dll"):
+        cmd += ["-shared"]
+    elif output.endswith(".obj"):
+        cmd += ["-c"]
+
     if isinstance(objects, str):
         objects = [objects]
-    cl_cmd += objects
+    cmd += ["-o", output]
+    cmd += objects
     if options:
-        cl_cmd += options
-
-    temp = tempdir()
-    dllmain_path = temp.relpath("dllmain.cc")
-    with open(dllmain_path, "w") as dllmain_obj:
-        dllmain_obj.write(
-            "#include <windows.h>\
-BOOL APIENTRY DllMain( HMODULE hModule,\
-                       DWORD  ul_reason_for_call,\
-                       LPVOID lpReserved)\
-{return TRUE;}"
-        )
-
-    cl_cmd += [dllmain_path]
-
-    temp_path = dllmain_path.replace("dllmain.cc", "")
-    cl_cmd += ["-Fo:" + temp_path]
-    try:
-        proc = subprocess.Popen(cl_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        (out, _) = proc.communicate()
-    except FileNotFoundError:
-        raise RuntimeError(
-            "Can not find cl.exe," "please run this in Vistual Studio Command Prompt."
-        )
-    if proc.returncode != 0:
-        msg = "Compilation error:\n"
-        msg += py_str(out)
-        raise RuntimeError(msg)
-    link_cmd = ["lld-link"]
-    link_cmd += ["-dll", "-FORCE:MULTIPLE"]
-
-    for obj in objects:
-        if obj.endswith(".cc"):
-            (_, temp_file_name) = os.path.split(obj)
-            (shot_name, _) = os.path.splitext(temp_file_name)
-            link_cmd += [os.path.join(temp_path, shot_name + ".obj")]
-        if obj.endswith(".o"):
-            link_cmd += [obj]
-
-    link_cmd += [temp_path + "dllmain.obj"]
-    link_cmd += ["-out:" + output]
+        cmd += options
 
     try:
-        proc = subprocess.Popen(link_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         (out, _) = proc.communicate()
     except FileNotFoundError:
         raise RuntimeError(
-            "Can not find the LLVM linker for Windows (lld-link.exe)."
+            "Can not find the LLVM clang for Windows clang.exe)."
             "Make sure it's installed"
             " and the installation directory is in the %PATH% environment "
             "variable. Prebuilt binaries can be found at: https://llvm.org/"
-            "For building the linker on your own see: https://lld.llvm.org/#build"
         )
     if proc.returncode != 0:
         msg = "Compilation error:\n"
index f260e29..1791a55 100644 (file)
@@ -25,7 +25,7 @@
 #define TVM_TARGET_LLVM_LLVM_COMMON_H_
 
 #ifdef _MSC_VER
-#pragma warning(disable : 4141 4291)
+#pragma warning(disable : 4141 4291 4146 4624)
 #endif
 
 #ifdef TVM_LLVM_VERSION
index 4513819..bb0a4e0 100644 (file)
@@ -159,20 +159,23 @@ def test_rpc_echo():
     check(rpc.LocalSession())
 
     check(client)
-    # Test minrpc server.
-    temp = util.tempdir()
-    minrpc_exec = temp.relpath("minrpc")
-    tvm.rpc.with_minrpc(cc.create_executable)(minrpc_exec, [])
-    check(rpc.PopenSession(minrpc_exec))
-    # minrpc on the remote
-    server = rpc.Server("localhost")
-    client = rpc.connect(
-        server.host,
-        server.port,
-        session_constructor_args=["rpc.PopenSession", open(minrpc_exec, "rb").read()],
-    )
-    check(client)
-
+    def check_minrpc():
+        if tvm.get_global_func("rpc.CreatePipeClient", allow_missing=True) is None:
+            return
+        # Test minrpc server.
+        temp = util.tempdir()
+        minrpc_exec = temp.relpath("minrpc")
+        tvm.rpc.with_minrpc(cc.create_executable)(minrpc_exec, [])
+        check(rpc.PopenSession(minrpc_exec))
+        # minrpc on the remote
+        server = rpc.Server("localhost")
+        client = rpc.connect(
+            server.host,
+            server.port,
+            session_constructor_args=["rpc.PopenSession", open(minrpc_exec, "rb").read()],
+        )
+        check(client)
+    check_minrpc()
 
 def test_rpc_file_exchange():
     if not tvm.runtime.enabled("rpc"):
@@ -234,7 +237,7 @@ def test_rpc_remote_module():
         np.testing.assert_equal(b.asnumpy(), a.asnumpy() + 1)
 
     def check_minrpc():
-        if tvm.get_global_func("rpc.PopenSession", allow_missing=True) is None:
+        if tvm.get_global_func("rpc.CreatePipeClient", allow_missing=True) is None:
             return
         # export to minrpc
         temp = util.tempdir()