[tfinfo] Trimming each token before processing (#9249)

This enables tfinfo to trim each token before processing.

Signed-off-by: Hyun Sik Yoon <hyunsik.yoon@samsung.com>

diff --git a/compiler/tfinfo/src/TensorInfoParser.cpp b/compiler/tfinfo/src/TensorInfoParser.cpp
index a28477e..4021a74 100644 (file)
--- a/compiler/tfinfo/src/TensorInfoParser.cpp
+++ b/compiler/tfinfo/src/TensorInfoParser.cpp
@@ -40,15 +40,26 @@ namespace
 
 using nnkit::support::tftestinfo::ParsedTensor;
 
-// remove comment and whitespaces
-void compact(std::string &line)
+// remove comment
+void remove_comment(std::string &line)
 {
   int pos = line.find_first_of("#");
   if (pos != std::string::npos)
     line.erase(pos);
+}
+
+std::string trim(const std::string &str)
+{
+  static const std::string whitespace = " \t";
+  static const std::string empty = "";
 
-  while ((pos = line.find_first_of(" \t\n\r")) != std::string::npos)
-    line.erase(pos, 1);
+  const auto begin = str.find_first_not_of(whitespace);
+  if (begin == std::string::npos)
+    return empty;
+
+  const auto end = str.find_last_not_of(whitespace);
+
+  return str.substr(begin, end - begin + 1);
 }
 
 ParsedTensor::Kind get_kind(const std::string &tok)
@@ -137,29 +148,31 @@ std::unique_ptr<ParsedTensor> parse_line(std::string &line)
   TF_DataType dtype;
   std::vector<int32_t> shape;
 
-  compact(line);
+  remove_comment(line);
 
   if (line.length() == 0) // empty line or line with comment
     return nullptr;
 
-  std::string tok, dim;
+  std::string tok, trimmed, dim;
 
   std::istringstream line_stream(line);
 
   CHECK_NOT_NULL(std::getline(line_stream, tok, ',')); // kind
-  kind = get_kind(tok);
+  kind = get_kind(trim(tok));
 
   CHECK_NOT_NULL(std::getline(line_stream, tok, ',')); // tensor name
-  if (!validate_name(tok))
+  trimmed = trim(tok);
+  if (!validate_name(trimmed))
     throw std::runtime_error("tensor name in wrong format: " + tok);
-  name.assign(tok);
+  name.assign(trimmed);
 
   CHECK_NOT_NULL(std::getline(line_stream, tok, ',')); // data type
-  dtype = get_dtype(tok);
+  dtype = get_dtype(trim(tok));
 
   CHECK_NOT_NULL(std::getline(line_stream, tok, '[')); // start of shape
-  if (tok.length())
-    throw std::runtime_error("unknown token between data type and shape: " + tok);
+  trimmed = trim(tok);
+  if (trimmed.length())
+    throw std::runtime_error("unknown token between data type and shape: " + trimmed);
 
   CHECK_NOT_NULL(std::getline(line_stream, tok, ']'));
 
@@ -168,6 +181,8 @@ std::unique_ptr<ParsedTensor> parse_line(std::string &line)
   bool first = true;
   while (std::getline(shape_stream, dim, ',')) // each dim
   {
+    dim = trim(dim);
+
     if (first && dim.length() == 0)
       continue; // scalar
     first = false;

diff --git a/compiler/tfinfo/src/TensorInfoParser.test.cpp b/compiler/tfinfo/src/TensorInfoParser.test.cpp
index 0cd6039..73ec61c 100644 (file)
--- a/compiler/tfinfo/src/TensorInfoParser.test.cpp
+++ b/compiler/tfinfo/src/TensorInfoParser.test.cpp
@@ -83,12 +83,23 @@ TEST(NNKIT_TF_PARSER, failure_case)
      "input, aa, TF_FLOAT, [abc]", // wrong name
      "input, a$a:0, TF_FLOAT, [abc]", // wrong name
      "input, aa:a, TF_FLOAT, [abc]", // wrong name (wrong value index)
+     "input aa:a, TF_FLOAT, [1]", // missing comma, exception.what() is "A line must be either 'input' or 'output' but : input aa:a"
+     "input, aa:a TF_FLOAT, [1]", // missing comma
+     "input, aa:a, TF_FLOAT [1]", // missing comma,
   };
   // clang-format on
 
   for (auto tc : exception_list)
   {
-    EXPECT_THROW(parse_line(tc), std::exception);
+    try
+    {
+      parse_line(tc);
+      FAIL();
+    }
+    catch (const std::exception &e)
+    {
+      std::cout << e.what() << '\n';
+    }
   }
 }