[Dist] use Tizen's zlib, protobuf, grpc
authorMyungJoo Ham <myungjoo.ham@samsung.com>
Wed, 10 Jul 2019 02:10:49 +0000 (11:10 +0900)
committerMyungJoo Ham <myungjoo.ham@samsung.com>
Wed, 10 Jul 2019 08:59:46 +0000 (17:59 +0900)
Let's use the system's protobuf, grpc, and zlib.
And do not export headers of protobuf from tensorflow.

Change-Id: I6ad41fd1a15b6d2651a94f5a098114000520f456
Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>
packaging/tensorflow.spec
tensorflow/contrib/cmake/CMakeLists.txt
tensorflow/contrib/cmake/external/grpc.cmake
tensorflow/contrib/cmake/external/protobuf.cmake
tensorflow/contrib/cmake/external/zlib.cmake

index ca736d66a2bd325bb2d528aba86ad8866a1b0cb6..77b9d2a28c3cc9be551f08878b5f9e1862735236 100644 (file)
@@ -13,34 +13,15 @@ Source10040:        farmhash.tar.gz
 Source10050:   fft2d.tar.gz
 Source10060:   gemmlowp.tar.gz
 Source10070:   gif.tar.gz
-Source20010:    grpc.tar.gz.00
-Source20011:    grpc.tar.gz.01
-Source20012:    grpc.tar.gz.02
-Source20013:    grpc.tar.gz.03
-Source20014:    grpc.tar.gz.04
-Source20015:    grpc.tar.gz.05
-Source20016:    grpc.tar.gz.06
-Source20017:    grpc.tar.gz.07
-Source20018:    grpc.tar.gz.08
-Source20019:    grpc.tar.gz.09
-Source20020:    grpc.tar.gz.10
-Source20021:    grpc.tar.gz.11
-Source20022:    grpc.tar.gz.12
-Source20023:    grpc.tar.gz.13
-Source20024:    grpc.tar.gz.14
-Source20025:    grpc.tar.gz.15
 Source30010:   highwayhash.tar.gz
 Source30020:   jpeg.tar.gz
 Source30030:   jsoncpp.tar.gz
 Source30040:   lmdb.tar.gz
 Source30050:   nsync.tar.gz
 Source30060:   png.tar.gz
-Source30070:   protobuf.tar.gz.00
-Source30071:   protobuf.tar.gz.01
 Source30080:   re2.tar.gz
 Source30090:   snappy.tar.gz
 Source30100:   sqlite.tar.gz
-Source30110:   zlib.tar.gz
 Source31000:   eigen.patch
 
 # Exclusively for tf-lite
@@ -66,6 +47,16 @@ BuildRequires:  python-numpy
 BuildRequires:  python-numpy-devel
 BuildRequires:  python-devel
 
+BuildRequires: pkgconfig(protobuf)
+BuildRequires: protobuf-compiler
+BuildRequires: protobuf-static
+BuildRequires: zlib-devel-static
+BuildRequires: pkgconfig(grpc)
+BuildRequires: grpc-devel-static
+
+# grpc requires libcares
+BuildRequires:  pkgconfig(libcares)
+
 %description
 TensorFlow is an open source software library for numerical computation using
 data flow graphs. Nodes in the graph represent mathematical operations, while
@@ -164,15 +155,6 @@ pushd gif/src
 tar -xf %{SOURCE10070}
 popd
 
-cat %{SOURCE20010} %{SOURCE20011} %{SOURCE20012} %{SOURCE20013} %{SOURCE20014} \
-       %{SOURCE20015} %{SOURCE20016} %{SOURCE20017} %{SOURCE20018} %{SOURCE20019} \
-       %{SOURCE20020} %{SOURCE20021} %{SOURCE20022} %{SOURCE20023} %{SOURCE20024} \
-       %{SOURCE20025} > grpc.tar.gz
-mkdir -p grpc/src
-pushd grpc/src
-tar -xf ../../grpc.tar.gz
-popd
-
 mkdir -p highwayhash/src
 pushd highwayhash/src
 tar -xf %{SOURCE30010}
@@ -203,12 +185,6 @@ pushd png/src
 tar -xf %{SOURCE30060}
 popd
 
-cat %{SOURCE30070} %{SOURCE30071} > protobuf.tar.gz
-mkdir -p protobuf/src
-pushd protobuf/src
-tar -xf ../../protobuf.tar.gz
-popd
-
 mkdir -p re2/src
 pushd re2/src
 tar -xf %{SOURCE30080}
@@ -224,10 +200,6 @@ pushd sqlite/src
 tar -xf %{SOURCE30100}
 popd
 
-mkdir -p zlib/src
-pushd zlib/src
-tar -xf %{SOURCE30110}
-popd
 %endif
 
 popd
@@ -270,7 +242,7 @@ pushd build
 # Seems some arguments of %%cmake is causing errors.
 # -Dtensorflow_ENABLE_SSL_SUPPORT=OFF  Cut the dependency to Go (not available in Tizen)
 # Use -DCMAKE_INSTALL_LIBDIR=lib because of discrepency in png install path.
-cmake ../tensorflow/contrib/cmake -DCMAKE_INSTALL_PREFIX=/usr -DLIB_INSTALL_DIR=/usr/lib -DCMAKE_INSTALL_LIBDIR=lib -DINCLUDE_INSTALL_DIR=/usr/include -Dtensorflow_ENABLE_SSL_SUPPORT=OFF
+cmake ../tensorflow/contrib/cmake -DCMAKE_INSTALL_PREFIX=/usr -DLIB_INSTALL_DIR=/usr/lib -DCMAKE_INSTALL_LIBDIR=lib -DINCLUDE_INSTALL_DIR=/usr/include -Dtensorflow_ENABLE_SSL_SUPPORT=OFF -Dsystemlib_PROTOBUF=ON -Dsystemlib_ZLIB=ON -Dsystemlib_GRPC=ON
 
 # QEMU unsupported syscall error is being writtin into .cmake files
 pushd CMakeFiles/pywrap_tensorflow_internal.dir
@@ -368,19 +340,9 @@ pushd build
 find tensorflow -name "*.h" -exec dirname {} \; | uniq | xargs -I {} bash -c 'if [ ! -d %{buildroot}%{_includedir}/%{name}/{} ] ; then mkdir %{buildroot}%{_includedir}/%{name}/{} -p ; fi'
 find tensorflow -name "*.h" -exec cp {} %{buildroot}%{_includedir}/%{name}/{} -aPv \;
 
-pushd protobuf/src/protobuf/src
-find google -name "*.h" -exec dirname {} \; | uniq | xargs -I {} mkdir %{buildroot}%{_includedir}/%{name}/{} -p
-find google -name "*.h" -exec cp {} %{buildroot}%{_includedir}/%{name}/{} -aPv \;
-popd
-
 cp -R -p eigen/src/eigen/Eigen %{buildroot}%{_includedir}/%{name}/
 cp -R -p eigen/src/eigen %{buildroot}%{_includedir}/%{name}/
 
-pushd protobuf/src/protobuf/src
-find google -name "*.h" -exec dirname {} \; | uniq | xargs -I {} mkdir %{buildroot}%{_includedir}/%{name}/{} -p
-find google -name "*.h" -exec cp {} %{buildroot}%{_includedir}/%{name}/{} -aPv \;
-popd
-
 pushd eigen/src/eigen
 find . -type f ! -name '*.cc' ! -name '*.cpp' ! -name '*.c' -exec dirname {} \; | uniq | xargs -I {} bash -c 'if [ ! -d %{buildroot}%{_includedir}/%{name}/third_party/eigen3/{} ] ; then mkdir %{buildroot}%{_includedir}/%{name}/third_party/eigen3/{} -p ; fi'
 find . -type f ! -name  '*.cc' ! -name '*.cpp' ! -name '*.c' -exec cp {} %{buildroot}%{_includedir}/%{name}/third_party/eigen3/{} -aPv \;
index c3de5b8421cbd671d710de557b4f169944725e53..fc412773cf5224c7552ab96a8e6602416b1f7d72 100644 (file)
@@ -111,12 +111,14 @@ if (NOT WIN32)
   option(systemlib_ZLIB "Use the system installed library as shared objects instead of downloading ZLIB and statically linking to it: ZLIB" OFF)
   option(systemlib_ABSEIL_CPP "Use the system installed library as shared objects instead of downloading ABSEIL_CPP and statically linking to it: ABSEIL_CPP" OFF)
   option(systemlib_PROTOBUF "Use the system installed library as shared objects instead of downloading PROTOBUF and statically linking to it: PROTOBUF" OFF)
+  option(systemlib_GRPC "Use the system installed library as shared objects instead of downloading GRPC and statically linking to it: GRPC" OFF)
 
   option(systemlib_ALL "Turn on every possible systemlib_* options" OFF)
   if (systemlib_ALL)
     set (systemlib_ZLIB ON)
     set (systemlib_ABSEIL_CPP ON)
     set (systemlib_PROTOBUF ON)
+    set (systemlib_GRPC ON)
   endif (systemlib_ALL)
 endif()
 
@@ -282,7 +284,6 @@ set(tensorflow_EXTERNAL_LIBRARIES
     ${fft2d_STATIC_LIBRARIES}
     ${highwayhash_STATIC_LIBRARIES}
     ${nsync_STATIC_LIBRARIES}
-    ${protobuf_STATIC_LIBRARIES}
     ${re2_STATIC_LIBRARIES}
     ${sqlite_STATIC_LIBRARIES}
     ${double_conversion_STATIC_LIBRARIES}
@@ -290,12 +291,20 @@ set(tensorflow_EXTERNAL_LIBRARIES
 
 if (systemlib_ZLIB)
   set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
-      ${ZLIB_LIBRARIES})
+    ${ZLIB_LIBRARIES})
 else (systemlib_ZLIB)
   set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
     ${zlib_STATIC_LIBRARIES})
 endif (systemlib_ZLIB)
 
+if (protobuf_PROTOBUF)
+  set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
+    ${PROTOBUF_LIBRARIES})
+else (protobuf_PROTOBUF)
+  set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
+    ${protobuf_STATIC_LIBRARIES})
+endif (protobuf_PROTOBUF)
+
 if (systemlib_ABSEIL_CPP)
   set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
       ${abseil_cpp_LIBRARIES})
@@ -304,13 +313,13 @@ else (systemlib_ABSEIL_CPP)
     ${abseil_cpp_STATIC_LIBRARIES})
 endif (systemlib_ABSEIL_CPP)
 
-if (systemlib_PROTOBUF)
+if (systemlib_GRPC)
   set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
-      ${PROTOBUF_LIBRARIES})
-else (systemlib_PROTOBUF)
+    ${GRPC_LIBRARIES} ${CARES_LIBRARIES})
+else (systemlib_GRPC)
   set(tensorflow_EXTERNAL_LIBRARIES ${tensorflow_EXTERNAL_LIBRARIES}
-    ${zlib_STATIC_LIBRARIES})
-endif (systemlib_PROTOBUF)
+    ${grpc_STATIC_LIBRARIES})
+endif (systemlib_GRPC)
 
 set(tensorflow_EXTERNAL_DEPENDENCIES
     zlib_copy_headers_to_destination
index e6eaa86bd085c948fb28d5a78451338f6659f4ba..bb60c90717e20b549fdccbad0d9afcc155d24b0b 100644 (file)
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-include (ExternalProject)
+if (systemlib_GRPC)
+  find_package(PkgConfig)
+  pkg_search_module(GRPC REQUIRED grpc)
+  set(grpc_INCLUDE_DIR ${GRPC_INCLUDE_DIRS})
+  set(ADD_LINK_DIRECTORY ${ADD_LINK_DIRECTORY} ${GRPC_LIBRARY_DIRS})
+  set(ADD_CFLAGS ${ADD_CFLAGS} ${GRPC_CFLAGS_OTHER})
+  set(grpc_STATIC_LIBRARIES libgrpc.a libgrpc++.a libaddress_sorting.a libgpr.a)
+  # To meet DEPENDS grpc from other projects.
+  # If we hit this line, grpc is already built and installed to the system.
+  add_custom_target(grpc)
+  add_custom_target(grpc_copy_headers_to_destination)
 
-set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
-set(GRPC_URL https://github.com/grpc/grpc.git)
-set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
-set(GRPC_TAG 69b6c047bc767b4d80e7af4d00ccb7c45b683dae)
+  # Locate grpc's executables for tf_core_framework.cmake.
+  set(GRPC_BUILD /usr/bin)
 
-if(WIN32)
-  # We use unsecure gRPC because boringssl does not build on windows
-  set(grpc_TARGET grpc++_unsecure)
-  set(grpc_DEPENDS protobuf zlib)
-  set(grpc_SSL_PROVIDER NONE)
-  if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
-    set(grpc_STATIC_LIBRARIES
-        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/grpc++_unsecure.lib
-        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/grpc_unsecure.lib
-        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/gpr.lib)
-  else()
-    set(grpc_STATIC_LIBRARIES
-        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/grpc++_unsecure.lib
-        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/grpc_unsecure.lib
-        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/gpr.lib)
-  endif()
-else()
-  set(grpc_TARGET grpc++)
-  if(tensorflow_ENABLE_SSL_SUPPORT)
-    set(grpc_DEPENDS boringssl protobuf zlib)
-    set(grpc_SSL_PROVIDER module)
-  else()
+  pkg_search_module(CARES REQUIRED libcares)
+  set(ADD_CFLAGS ${ADD_CFLAGS} ${CARES_CFLAGS_OTHER})
+
+else (systemlib_GRPC)
+  include (ExternalProject)
+
+  set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
+  set(GRPC_URL https://github.com/grpc/grpc.git)
+  set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
+  set(GRPC_TAG 69b6c047bc767b4d80e7af4d00ccb7c45b683dae)
+
+  if(WIN32)
+    # We use unsecure gRPC because boringssl does not build on windows
+    set(grpc_TARGET grpc++_unsecure)
     set(grpc_DEPENDS protobuf zlib)
     set(grpc_SSL_PROVIDER NONE)
+    if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
+      set(grpc_STATIC_LIBRARIES
+          ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/grpc++_unsecure.lib
+          ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/grpc_unsecure.lib
+          ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/$(Configuration)/gpr.lib)
+    else()
+      set(grpc_STATIC_LIBRARIES
+          ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/grpc++_unsecure.lib
+          ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/grpc_unsecure.lib
+          ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/gpr.lib)
+    endif()
+  else()
+    set(grpc_TARGET grpc++)
+    if(tensorflow_ENABLE_SSL_SUPPORT)
+      set(grpc_DEPENDS boringssl protobuf zlib)
+      set(grpc_SSL_PROVIDER module)
+    else()
+      set(grpc_DEPENDS protobuf zlib)
+      set(grpc_SSL_PROVIDER NONE)
+    endif()
+    set(grpc_STATIC_LIBRARIES
+        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++.a
+        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc.a
+        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libaddress_sorting.a
+        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a
+        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a
+        ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/zlib/libz.a)
   endif()
-  set(grpc_STATIC_LIBRARIES
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++.a
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc.a
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libaddress_sorting.a
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/zlib/libz.a)
-endif()
 
-add_definitions(-DGRPC_ARES=0)
+  add_definitions(-DGRPC_ARES=0)
 
-ExternalProject_Add(grpc
-    PREFIX grpc
-    DEPENDS ${grpc_DEPENDS}
-    GIT_REPOSITORY ${GRPC_URL}
-    GIT_TAG ${GRPC_TAG}
-    DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
-    BUILD_IN_SOURCE 1
-    BUILD_BYPRODUCTS ${grpc_STATIC_LIBRARIES}
-    BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release --target ${grpc_TARGET}
-    COMMAND ${CMAKE_COMMAND} --build . --config Release --target grpc_cpp_plugin
-    INSTALL_COMMAND ""
-    CMAKE_CACHE_ARGS
-        -DCMAKE_BUILD_TYPE:STRING=Release
-        -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-        -DPROTOBUF_INCLUDE_DIRS:STRING=${PROTOBUF_INCLUDE_DIRS}
-        -DPROTOBUF_LIBRARIES:STRING=${protobuf_STATIC_LIBRARIES}
-        -DZLIB_ROOT:STRING=${ZLIB_INSTALL}
-        -DgRPC_SSL_PROVIDER:STRING=${grpc_SSL_PROVIDER}
-)
+  ExternalProject_Add(grpc
+      PREFIX grpc
+      DEPENDS ${grpc_DEPENDS}
+      GIT_REPOSITORY ${GRPC_URL}
+      GIT_TAG ${GRPC_TAG}
+      DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+      BUILD_IN_SOURCE 1
+      BUILD_BYPRODUCTS ${grpc_STATIC_LIBRARIES}
+      BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release --target ${grpc_TARGET}
+      COMMAND ${CMAKE_COMMAND} --build . --config Release --target grpc_cpp_plugin
+      INSTALL_COMMAND ""
+      CMAKE_CACHE_ARGS
+          -DCMAKE_BUILD_TYPE:STRING=Release
+          -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+          -DPROTOBUF_INCLUDE_DIRS:STRING=${PROTOBUF_INCLUDE_DIRS}
+          -DPROTOBUF_LIBRARIES:STRING=${protobuf_STATIC_LIBRARIES}
+          -DZLIB_ROOT:STRING=${ZLIB_INSTALL}
+          -DgRPC_SSL_PROVIDER:STRING=${grpc_SSL_PROVIDER}
+  )
 
-# grpc/src/core/ext/census/tracing.c depends on the existence of openssl/rand.h.
-ExternalProject_Add_Step(grpc copy_rand
-    COMMAND ${CMAKE_COMMAND} -E copy
-    ${CMAKE_SOURCE_DIR}/patches/grpc/rand.h ${GRPC_BUILD}/include/openssl/rand.h
-    DEPENDEES patch
-    DEPENDERS build
-)
+  # grpc/src/core/ext/census/tracing.c depends on the existence of openssl/rand.h.
+  ExternalProject_Add_Step(grpc copy_rand
+      COMMAND ${CMAKE_COMMAND} -E copy
+      ${CMAKE_SOURCE_DIR}/patches/grpc/rand.h ${GRPC_BUILD}/include/openssl/rand.h
+      DEPENDEES patch
+      DEPENDERS build
+  )
+endif (systemlib_GRPC)
index 18aa7cf9e8e455b604189112398e3d3b4b447d57..0bdeb66197e6d300e9d56c2affe1f1edf6c09e91 100644 (file)
 if (systemlib_PROTOBUF)
   find_package(PkgConfig)
   pkg_search_module(PROTOBUF REQUIRED protobuf)
+  set(PROTOBUF_INCLUDE_DIRS /usr/include ${PROTOBUF_INCLUDE_DIRS})
   set(protobuf_INCLUDE_DIR ${PROTOBUF_INCLUDE_DIRS})
   set(ADD_LINK_DIRECTORY ${ADD_LINK_DIRECTORY} ${PROTOBUF_LIBRARY_DIRS})
   set(ADD_CFLAGS ${ADD_CFLAGS} ${PROTOBUF_CFLAGS_OTHER})
+  set(protobuf_STATIC_LIBRARIES libprotobuf.a)
+  set(PROTOBUF_PROTOC_EXECUTABLE /usr/bin/protoc)
+
+  message("PROTOBUF_INCLUDE_DIRS : [${PROTOBUF_INCLUDE_DIRS}]")
 
   # To meet DEPENDS protobuf from other projects.
   # If we hit this line, protobuf is already built and installed to the system.
index 8942f3eecf07fff893884795a104422529357bf8..352283078014aaba959e0c200358c601d044c6fc 100644 (file)
@@ -18,6 +18,7 @@ if (systemlib_ZLIB)
   set(zlib_INCLUDE_DIR ${ZLIB_INCLUDE_DIRS})
   set(ADD_LINK_DIRECTORY ${ADD_LINK_DIRECTORY} ${ZLIB_LIBRARY_DIRS})
   set(ADD_CFLAGS ${ADD_CFLAGS} ${ZLIB_CFLAGS_OTHER})
+  set(zlib_STATIC_LIBRARIES libz.a)
 
   # To meet DEPENDS zlib from other projects.
   # If we hit this line, zlib is already built and installed to the system.