From: Vitaliy Cherepanov/AI Tools Lab /SRR/Engineer/삼성전자 <v.cherepanov@samsung.com>
Date: Mon, 16 Jul 2018 17:15:46 +0000 (+0300)
Subject: Code emitter: def files to C/C++ arrays converter (#673)
X-Git-Tag: nncc_backup~2411
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=10125478c938dc211d442d8a84b5c1ece6664337;p=platform%2Fcore%2Fml%2Fnnfw.git

Code emitter: def files to C/C++ arrays converter (#673)

add *.def to *.h converter

Signed-off-by: Vitaliy Cherepanov <v.cherepanov@samsung.com>
---

diff --git a/contrib/nnc/libs/backend/soft/CMakeLists.txt b/contrib/nnc/libs/backend/soft/CMakeLists.txt
index a71f1e4..4da485c 100644
--- a/contrib/nnc/libs/backend/soft/CMakeLists.txt
+++ b/contrib/nnc/libs/backend/soft/CMakeLists.txt
@@ -1,17 +1,42 @@
 file(GLOB_RECURSE SOFT_BACKEND_SOURCES src/*.cpp)
 file(GLOB_RECURSE HEADERS "include/*.h")
+set(DEF_CONV src/def2src.cpp)
+
 file(GLOB_RECURSE TESTS "src/*.test.cpp")
-list(REMOVE_ITEM SOFT_BACKEND_SOURCES ${TESTS})
+file(GLOB_RECURSE SOFT_DEF_SOURCES include/*.def)
+list(REMOVE_ITEM SOFT_BACKEND_SOURCES ${TESTS} ${DEF_CONV})
+
+# TODO move this macro to common make file
+macro(replace_ext ext dest_list src_list)
+    SET(${dest_list} "")
+    FOREACH(file IN LISTS ${src_list})
+        GET_FILENAME_COMPONENT(file_path ${file} PATH)
+        GET_FILENAME_COMPONENT(file_name ${file} NAME_WE)
+        list(APPEND ${dest_list} "${file_path}/${file_name}/${ext}")
+    ENDFOREACH()
+endmacro()
+
+replace_ext(".h" SOFT_GENERATED_SOURCES SOFT_DEF_SOURCES)
 
-add_library(soft_backend SHARED ${SOFT_BACKEND_SOURCES})
+add_library(soft_backend SHARED ${SOFT_BACKEND_SOURCES} ${SOFT_GENERATED_SOURCES})
 
 add_nncc_test(nnc_soft_backend_test ${TESTS})
 nncc_target_link_libraries(nnc_soft_backend_test nncc_core nnc_core nnc_plugin_core nncc_foundation soft_backend)
 target_include_directories(nnc_soft_backend_test PUBLIC include)
+target_include_directories(nnc_soft_backend_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+
+add_executable(def2src ${DEF_CONV})
+
+add_custom_command(
+        OUTPUT  ${SOFT_GENERATED_SOURCES}
+        COMMAND def2src ${CMAKE_CURRENT_BINARY_DIR} ${SOFT_DEF_SOURCES}
+        DEPENDS def2src ${SOFT_DEF_SOURCES}
+)
 
 target_link_libraries(soft_backend PRIVATE nncc_core)
 
 target_include_directories(soft_backend PUBLIC include)
+target_include_directories(soft_backend PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
 target_link_libraries(soft_backend PRIVATE nnc_plugin_core)
 target_link_libraries(soft_backend PRIVATE nncc_foundation)
diff --git a/contrib/nnc/libs/backend/soft/include/cpp_header_types.def b/contrib/nnc/libs/backend/soft/include/cpp_header_types.def
index 007be5c..356d597 100644
--- a/contrib/nnc/libs/backend/soft/include/cpp_header_types.def
+++ b/contrib/nnc/libs/backend/soft/include/cpp_header_types.def
@@ -1,157 +1,157 @@
-S(const int MAX_DIMS = 8;)
-S()
-S(template<int maxDims>)
-S(class BaseVector)
-S({)
-S(public:)
-S()
-S(  BaseVector())
-S(  {)
-S(    _dims = 0;)
-S(    _volume = 0;)
-S(  })
-S()
-S(  template <class T>)
-S(  BaseVector(std::initializer_list<T> data): _dims(data.size()))
-S(  {)
-S(    assert(_dims <= maxDims);)
-S(    size_t *dataPtr = _data;)
-S(    for (T value: data))
-S(    {)
-S(      *dataPtr++ = static_cast<size_t>(value);)
-S(      _volume *= static_cast<size_t>(value);)
-S(    })
-S(  })
-S()
-S(  BaseVector(const BaseVector &orig): _dims(orig._dims), _volume(orig._volume))
-S(  {)
-S(    for (int i = 0; i < _dims; ++i))
-S(      _data[i] = orig._data[i];)
-S(  })
-S()
-S(  BaseVector &operator=(const BaseVector &orig))
-S(  {)
-S(    _dims = orig._dims;)
-S(    _volume = orig._volume;)
-S(    for (int i = 0; i < _dims; ++i))
-S(      _data[i] = orig._data[i];)
-S(    return *this;)
-S(  })
-S()
-S(  void change(std::initializer_list<size_t> data))
-S(  {)
-S(    _dims = data.size();)
-S(    assert(_dims <= maxDims);)
-S(    _volume = 1;)
-S(    size_t *dataPtr = _data;)
-S(    for (size_t value: data))
-S(    {)
-S(      *dataPtr++ = value;)
-S(      _volume *= value;)
-S(    })
-S(  })
-S()
-S(  int dims() const)
-S(  {)
-S(    return _dims;)
-S(  })
-S()
-S(  size_t operator[](int dim) const)
-S(  {)
-S(    assert(dim < _dims);)
-S(    return _data[dim];)
-S(  })
-S()
-S(  size_t getVolume() const)
-S(  {)
-S(    return _volume;)
-S(  })
-S()
-S(private:)
-S(  size_t _data[maxDims];)
-S(  int _dims;)
-S(  size_t _volume = 1;)
-S(};)
-S()
-S(using Shape = BaseVector<MAX_DIMS>;)
-S()
-S(using Index = BaseVector<MAX_DIMS>;)
-S()
-S(class Tensor)
-S({)
-S(public:)
-S(  Tensor(): _shape(), _data(nullptr), _managed(true){})
-S()
-S(  Tensor(const Shape& shape, float *data): _shape(shape), _data(data){})
-S()
-S(  Tensor(const Shape& shape): _shape(shape), _data(new float[shape.getVolume()]), _managed(true) {})
-S()
-S(  ~Tensor())
-S(  {)
-S(    if (_managed))
-S(      delete [] _data;)
-S(  })
-S()
-S(  void fillData(const float *data))
-S(  {)
-S(    assert(_managed);)
-S(    for (int i = 0; i < _shape.getVolume(); ++i))
-S(      _data[i] = data[i];)
-S(  })
-S()
-S(  Tensor &operator=(const Tensor &t))
-S(  {)
-S(    assert(_managed);)
-S(    if (this != &t))
-S(    {)
-S(      reShape(t.getShape());)
-S(      fillData(t._data);)
-S(    })
-S(    return *this;)
-S(  })
-S()
-S(  float &at(const Index &idx))
-S(  {)
-S(    return *(_data + getOffset(idx));)
-S(  })
-S()
-S(  float at(const Index &idx) const)
-S(  {)
-S(    return *(_data + getOffset(idx));)
-S(  })
-S()
-S(  void reShape(const Shape &shape))
-S(  {)
-S(    size_t oldVolume = _shape.getVolume();)
-S(    _shape = shape;)
-S(    if (_managed && oldVolume != shape.getVolume()))
-S(    {)
-S(      delete [] _data;)
-S(      _data = new float[shape.getVolume()];)
-S(    })
-S(  })
-S()
-S(  const Shape &getShape() const)
-S(  {)
-S(    return _shape;)
-S(  })
-S()
-S(private:)
-S(  size_t getOffset(const Index &idx) const)
-S(  {)
-S(    assert(idx.dims() == _shape.dims());)
-S(    size_t offset = 0;)
-S(    size_t stride = 1;)
-S(    for (int i = _shape.dims() - 1; i >= 0; --i))
-S(    {)
-S(      assert(idx[i] < _shape[i]);)
-S(      offset += stride * idx[i];)
-S(      stride *= _shape[i];)
-S(    })
-S(    return offset;)
-S(  })
-S()
-S(  Shape _shape;)
-S(  float *_data;)
-S(  bool _managed = false;)
-S(};)
+const int MAX_DIMS = 8;
+
+template<int maxDims>
+class BaseVector
+{
+public:
+
+  BaseVector()
+  {
+    _dims = 0;
+    _volume = 0;
+  }
+
+  template <class T>
+  BaseVector(std::initializer_list<T> data): _dims(data.size())
+  {
+    assert(_dims <= maxDims);
+    size_t *dataPtr = _data;
+    for (T value: data)
+    {
+      *dataPtr++ = static_cast<size_t>(value);
+      _volume *= static_cast<size_t>(value);
+    }
+  }
+
+  BaseVector(const BaseVector &orig): _dims(orig._dims), _volume(orig._volume)
+  {
+    for (int i = 0; i < _dims; ++i)
+      _data[i] = orig._data[i];
+  }
+
+  BaseVector &operator=(const BaseVector &orig)
+  {
+    _dims = orig._dims;
+    _volume = orig._volume;
+    for (int i = 0; i < _dims; ++i)
+      _data[i] = orig._data[i];
+    return *this;
+  }
+
+  void change(std::initializer_list<size_t> data)
+  {
+    _dims = data.size();
+    assert(_dims <= maxDims);
+    _volume = 1;
+    size_t *dataPtr = _data;
+    for (size_t value: data)
+    {
+      *dataPtr++ = value;
+      _volume *= value;
+    }
+  }
+
+  int dims() const
+  {
+    return _dims;
+  }
+
+  size_t operator[](int dim) const
+  {
+    assert(dim < _dims);
+    return _data[dim];
+  }
+
+  size_t getVolume() const
+  {
+    return _volume;
+  }
+
+private:
+  size_t _data[maxDims];
+  int _dims;
+  size_t _volume = 1;
+};
+
+using Shape = BaseVector<MAX_DIMS>;
+
+using Index = BaseVector<MAX_DIMS>;
+
+class Tensor
+{
+public:
+  Tensor(): _shape(), _data(nullptr), _managed(true){}
+
+  Tensor(const Shape& shape, float *data): _shape(shape), _data(data){}
+
+  Tensor(const Shape& shape): _shape(shape), _data(new float[shape.getVolume()]), _managed(true) {}
+
+  ~Tensor()
+  {
+    if (_managed)
+      delete [] _data;
+  }
+
+  void fillData(const float *data)
+  {
+    assert(_managed);
+    for (int i = 0; i < _shape.getVolume(); ++i)
+      _data[i] = data[i];
+  }
+
+  Tensor &operator=(const Tensor &t)
+  {
+    assert(_managed);
+    if (this != &t)
+    {
+      reShape(t.getShape());
+      fillData(t._data);
+    }
+    return *this;
+  }
+
+  float &at(const Index &idx)
+  {
+    return *(_data + getOffset(idx));
+  }
+
+  float at(const Index &idx) const
+  {
+    return *(_data + getOffset(idx));
+  }
+
+  void reShape(const Shape &shape)
+  {
+    size_t oldVolume = _shape.getVolume();
+    _shape = shape;
+    if (_managed && oldVolume != shape.getVolume())
+    {
+      delete [] _data;
+      _data = new float[shape.getVolume()];
+    }
+  }
+
+  const Shape &getShape() const
+  {
+    return _shape;
+  }
+
+private:
+  size_t getOffset(const Index &idx) const
+  {
+    assert(idx.dims() == _shape.dims());
+    size_t offset = 0;
+    size_t stride = 1;
+    for (int i = _shape.dims() - 1; i >= 0; --i)
+    {
+      assert(idx[i] < _shape[i]);
+      offset += stride * idx[i];
+      stride *= _shape[i];
+    }
+    return offset;
+  }
+
+  Shape _shape;
+  float *_data;
+  bool _managed = false;
+};
diff --git a/contrib/nnc/libs/backend/soft/include/cpp_operations.def b/contrib/nnc/libs/backend/soft/include/cpp_operations.def
index cd6cab7..e69de29 100644
--- a/contrib/nnc/libs/backend/soft/include/cpp_operations.def
+++ b/contrib/nnc/libs/backend/soft/include/cpp_operations.def
@@ -1 +0,0 @@
-S()
diff --git a/contrib/nnc/libs/backend/soft/src/cpp_header_types.test.cpp b/contrib/nnc/libs/backend/soft/src/cpp_header_types.test.cpp
index 9ac59cf..5994c81 100644
--- a/contrib/nnc/libs/backend/soft/src/cpp_header_types.test.cpp
+++ b/contrib/nnc/libs/backend/soft/src/cpp_header_types.test.cpp
@@ -2,7 +2,7 @@
 #include <cassert>
 #include <initializer_list>
 #include <numeric>
-#define S(...) __VA_ARGS__
+
 #include <cpp_header_types.def>
 
 #include "gtest/gtest.h"
diff --git a/contrib/nnc/libs/backend/soft/src/def2src.cpp b/contrib/nnc/libs/backend/soft/src/def2src.cpp
new file mode 100644
index 0000000..dd67d16
--- /dev/null
+++ b/contrib/nnc/libs/backend/soft/src/def2src.cpp
@@ -0,0 +1,70 @@
+#include <iostream>
+#include <fstream>
+
+int fileToArray(std::string &source, std::string &dest, std::string arrName) {
+  FILE *fs = fopen(source.c_str(), "rb");
+  if (!fs) {
+    std::cerr << "source file not found: <" << source << ">" << std::endl;
+    return -1;
+  }
+
+  std::ofstream fo(dest.c_str());
+  if (fo.fail()) {
+    std::cerr << "cannot generate file: <" << dest << ">" << std::endl;
+    fclose(fs);
+    return -1;
+  }
+
+  std::cout << "generating <" << dest << ">" << std::endl;
+
+  fo << "const char " << arrName << "[] = {" << std::endl;
+
+  fseek(fs, 0, SEEK_SET);
+  size_t bytes;
+  do {
+    char buf[1024];
+    bytes = fread(buf, 1, sizeof(buf), fs);
+
+    // convert line
+    for (size_t i = 0; i < bytes; i++) {
+      fo << "0x" << std::hex << (int)buf[i] << ", ";
+    }
+  } while (bytes != 0);
+
+  fo << "};" << std::endl;
+  fo.flush();
+  fclose(fs);
+
+  return 0;
+}
+
+std::string extractFileName(std::string path)
+{
+  auto pos = path.find_last_of('/');
+  if (pos != std::string::npos)
+    path = path.substr(pos + 1);
+
+  pos = path.find_first_of('.');
+  if (pos != std::string::npos)
+    path = path.substr(0, pos);
+
+  return path;
+}
+
+int main(int argc, char *argv[]) {
+  if (argc < 3)
+    return -1;
+
+  std::string OutPutDir = argv[1];
+
+  for (int i = 2; i < argc; i++) {
+    std::string sourceFullFileName = argv[i];
+    std::string filename = extractFileName(sourceFullFileName);
+    std::string outputFileName = OutPutDir + "/" + filename + ".h";
+
+    if (fileToArray(sourceFullFileName, outputFileName, filename) != 0)
+      return -1;
+  }
+
+  return 0;
+}
diff --git a/contrib/nnc/libs/backend/soft/src/generator.cpp b/contrib/nnc/libs/backend/soft/src/generator.cpp
index a0d041e..aa4a4f3 100644
--- a/contrib/nnc/libs/backend/soft/src/generator.cpp
+++ b/contrib/nnc/libs/backend/soft/src/generator.cpp
@@ -17,6 +17,9 @@ using namespace std;
 using namespace nncc::contrib;
 using namespace nncc::contrib::core::IR::model;
 
+#include "cpp_header_types.h"
+#include "cpp_operations.h"
+
 namespace nncc
 {
 namespace contrib
@@ -188,20 +191,6 @@ CPPCodeGenerator CPPCodeGenerator::create(const std::string &headerFile,
   return gen;
 }
 
-// put generator base types into string
-#ifdef S
-  #error S macro already defined!
-#endif
-#define S(...) #__VA_ARGS__ "\n"
-  static const char *cpp_header_types =
-  #include "cpp_header_types.def"
-  ;
-
-  static const char *cpp_operations =
-  #include "cpp_operations.def"
-  ;
-#undef S
-
 void CPPCodeGenerator::materializeHeader(ostream &out, const ModelAnalyzer &ma)
 {
   string className = ma.getModelName() + "Model";