Update input_tensor_info for configuring the input

[Problem] Input tensor information does not match to the configured input
[Solution] Update the input_tensor_info by configured input information

 * SetInputTensorInfo() will change the model's input tensor information,
   even though the caller has configured the input.
   In other words, The GetInputTensorInfo() will not be affected by SetInputTensorInfo()
   if the input data is configured.

 * GetInputTensorInfo() only be affected by the configured input

 - SetInputTensorInfo() -> Always tries to change the input tensor specification of the model
 - SetOutputTensorInfo() -> Always tries to change the output tensor specification of the model
 - GetOutputTensorInfo() -> Always returns the output tensor specification of the model.
 - GetInputTensorInfo() -> returns the input tensor specification.
                           If the input is configured, the configured input specification will be returned

* Use the configuration input tensor info
   It is not necessary to get the model's input tensor info
   if there is a configured input information

Change-Id: I313a0b9f3a5be4f8f0ae95b09be03ac76bc742d0
Signed-off-by: Sung-jae Park <nicesj.park@samsung.com>

diff --git a/subprojects/libbeyond-peer_nn/include/peer.h b/subprojects/libbeyond-peer_nn/include/peer.h
index 91182a2c7c37ead9937ab734555b058e9fada5e1..1da653b089069e6d52c2cba2e7b50f7d4cf3f270 100644 (file)
--- a/subprojects/libbeyond-peer_nn/include/peer.h
+++ b/subprojects/libbeyond-peer_nn/include/peer.h
@@ -19,6 +19,7 @@
 
 #include <memory>
 #include <string>
+#include <map>
 
 #include <beyond/platform/beyond_platform.h>
 #include <beyond/private/beyond_private.h>
@@ -94,6 +95,8 @@ private:
         char payload[1];
     };
 
+    static const std::map<std::string, int> sizeMap;
+
 private:
     Peer(void);
     virtual ~Peer(void);
@@ -105,12 +108,15 @@ private:
 
     static void ConfigureImageInput(beyond_input_config *config, std::ostringstream &client_format, std::ostringstream &server_format);
     static void ConfigureVideoInput(beyond_input_config *config, std::ostringstream &client_format, std::ostringstream &server_format);
+    static beyond_tensor_info *AllocInputTensorInfo(beyond_input_config *config);
 
     static void ResetInfo(beyond_peer_info *&info);
     static void ResetRuntime(beyond_peer_info_runtime *&runtimes, int count_of_runtimes);
     static beyond_plugin_peer_nn_config *DuplicateConfig(const beyond_plugin_peer_nn_config *config);
     static void FreeConfig(beyond_plugin_peer_nn_config *&config);
 
+    static void FreeInputTensorInfo(beyond_tensor_info *&info);
+
 private:
     std::unique_ptr<ServerContext> serverCtx;
     std::unique_ptr<ClientContext> clientCtx;
@@ -121,6 +127,8 @@ private:
     beyond_plugin_peer_nn_config *reservedConfiguration;
     beyond::AuthenticatorInterface *authenticator;
     beyond::AuthenticatorInterface *caAuthenticator;
+
+    beyond_tensor_info *configuredInputTensorInfo;
 };
 
 #endif // __BEYOND_PEER_NN_PEER_H__

diff --git a/subprojects/libbeyond-peer_nn/include/peer_model.h b/subprojects/libbeyond-peer_nn/include/peer_model.h
index 3ff1778fe2b0f0b50979a3050aeb7ecaf65a2dd8..11a6a43390aa8769c74e04a3a3487b0dc6710cca 100644 (file)
--- a/subprojects/libbeyond-peer_nn/include/peer_model.h
+++ b/subprojects/libbeyond-peer_nn/include/peer_model.h
@@ -31,6 +31,7 @@ public:
     int GetOutputTensorInfo(const beyond_tensor_info *&info, int &size);
     int SetInputTensorInfo(beyond_tensor_info *info, int size);
     int SetOutputTensorInfo(beyond_tensor_info *info, int size);
+    void InvalidateTensorInfo(void);
 
     static void FreeTensorInfo(beyond_tensor_info *&info, int &size);
     static int DupTensorInfo(beyond_tensor_info *&dest, const beyond_tensor_info *src, int size);

diff --git a/subprojects/libbeyond-peer_nn/src/peer.cc b/subprojects/libbeyond-peer_nn/src/peer.cc
index bb35d70990bd8af7dfcd108ecf0789291786ff4a..2616e81cd34025ce838c8ae7cd2e7cbc35c23642 100644 (file)
--- a/subprojects/libbeyond-peer_nn/src/peer.cc
+++ b/subprojects/libbeyond-peer_nn/src/peer.cc
@@ -35,6 +35,81 @@
 #include <unistd.h>
 
 #define DEFAULT_FRAMEWORK "tensorflow-lite"
+#define CONFIGURED_INPUT_TENSOR_SIZE 1
+#define CONFIGURED_INPUT_TENSOR_DIMS 2
+
+// TODO:
+// We have to update the sizeMap with more precise values for each of image format
+const std::map<std::string, int> Peer::sizeMap = {
+    {
+        "I420", // 640x480(Y), 320x240(U), 320x240(U) per a pixel
+        3,      // It is enough to represent YUV size in bytes
+    },
+    {
+        "YV12",
+        3,
+    },
+    {
+        "YUY2",
+        3,
+    },
+    {
+        "UYVY",
+        3,
+    },
+    {
+        "Y41B",
+        3,
+    },
+    {
+        "Y42B",
+        3,
+    },
+    {
+        "YVYU",
+        3,
+    },
+    {
+        "Y444",
+        3,
+    },
+    {
+        "NV21",
+        3,
+    },
+    {
+        "NV12",
+        3,
+    },
+    {
+        "RGB",
+        3,
+    },
+    {
+        "BGR",
+        3,
+    },
+    {
+        "RGBx",
+        4,
+    },
+    {
+        "xRGB",
+        4,
+    },
+    {
+        "BGRx",
+        4,
+    },
+    {
+        "xBGR",
+        4,
+    },
+    {
+        "GRAY8",
+        1,
+    }
+};
 
 Peer *Peer::Create(bool isServer, const char *framework, const char *accel, const char *storagePath)
 {
@@ -122,6 +197,8 @@ void Peer::Destroy(void)
     eventObject = nullptr;
 
     ResetInfo(info);
+    FreeConfig(reservedConfiguration);
+    FreeInputTensorInfo(configuredInputTensorInfo);
 
     delete this;
 }
@@ -205,6 +282,67 @@ void Peer::ConfigureVideoInput(beyond_input_config *config, std::ostringstream &
     server_format << " ! queue leaky=2 max-size-buffers=1";
 }
 
+beyond_tensor_info *Peer::AllocInputTensorInfo(beyond_input_config *config)
+{
+    const beyond_input_image_config *imageConfig = nullptr;
+
+    if (config->input_type == BEYOND_INPUT_TYPE_IMAGE) {
+        imageConfig = &config->config.image;
+    } else if (config->input_type == BEYOND_INPUT_TYPE_VIDEO) {
+        imageConfig = &config->config.video.frame;
+    } else {
+        ErrPrint("Unknown input type configuration");
+        return nullptr;
+    }
+
+    if (imageConfig->format == nullptr) {
+        ErrPrint("Format is not specified");
+        return nullptr;
+    }
+
+    std::map<std::string, int>::const_iterator it = sizeMap.find(imageConfig->format);
+    if (it == sizeMap.end()) {
+        ErrPrint("Unsupported format [%s]", imageConfig->format);
+        return nullptr;
+    }
+
+    beyond_tensor_info *info = static_cast<beyond_tensor_info *>(malloc(sizeof(beyond_tensor_info) * CONFIGURED_INPUT_TENSOR_SIZE));
+    if (info == nullptr) {
+        ErrPrintCode(errno, "malloc");
+        return nullptr;
+    }
+
+    info->size = imageConfig->width * imageConfig->height * it->second;
+    info->type = BEYOND_TENSOR_TYPE_UINT8;
+    info->name = strdup(imageConfig->format);
+    if (info->name == nullptr) {
+        ErrPrintCode(errno, "strdup");
+        free(info);
+        return nullptr;
+    }
+
+    // NOTE:
+    // WIDTH and HEIGHT, dimension size is always 2
+    info->dims = static_cast<beyond_tensor_info::dimensions *>(malloc(sizeof(beyond_tensor_info::dimensions) + sizeof(int) * CONFIGURED_INPUT_TENSOR_DIMS));
+    if (info->dims == nullptr) {
+        ErrPrintCode(errno, "malloc");
+        free(info->name);
+        info->name = nullptr;
+        free(info);
+        return nullptr;
+    }
+
+    // TODO:
+    // This array information should be updated for each image format specification
+    // In case of the YUV, it could be manipulatd as a 3 dimensional array buffer
+    // At least now, image format related buffer layout will be treated as a 2-dimensional
+    // array (width and height) buffer
+    info->dims->size = CONFIGURED_INPUT_TENSOR_DIMS;
+    info->dims->data[0] = imageConfig->width;
+    info->dims->data[1] = imageConfig->height;
+    return info;
+}
+
 int Peer::ConfigureInput(const beyond_config *options)
 {
     if (serverCtx != nullptr) {
@@ -279,6 +417,19 @@ int Peer::ConfigureInput(const beyond_config *options)
             return ret;
         }
 
+        beyond_tensor_info *newInfo = AllocInputTensorInfo(config);
+        if (newInfo == nullptr) {
+            int ret = -errno;
+            ErrPrintCode(errno, "strdup");
+            free(client_desc);
+            client_desc = nullptr;
+            free(server_desc);
+            server_desc = nullptr;
+            return ret;
+        }
+
+        FreeInputTensorInfo(configuredInputTensorInfo);
+        configuredInputTensorInfo = newInfo;
         _options->client.preprocessing = client_desc;
         _options->server.preprocessing = server_desc;
     } else {
@@ -393,6 +544,15 @@ int Peer::GetInputTensorInfo(const beyond_tensor_info *&info, int &size)
         return -EILSEQ;
     }
 
+    // NOTE:
+    // If there is a configured input information,
+    // GetInputTensorInfo() should return the configured input tensor information
+    if (configuredInputTensorInfo) {
+        size = 1;
+        info = configuredInputTensorInfo;
+        return 0;
+    }
+
     const beyond_tensor_info *_info = nullptr;
     int _size = 0;
 
@@ -514,6 +674,20 @@ int Peer::AllocateTensor(const beyond_tensor_info *info, int size, beyond_tensor
     return 0;
 }
 
+void Peer::FreeInputTensorInfo(beyond_tensor_info *&info)
+{
+    if (info == nullptr) {
+        return;
+    }
+
+    free(info->name);
+    info->name = nullptr;
+    free(info->dims);
+    info->dims = nullptr;
+    free(info);
+    info = nullptr;
+}
+
 void Peer::FreeTensor(beyond_tensor *&tensor, int size)
 {
     if (clientCtx == nullptr) {
@@ -1055,6 +1229,7 @@ Peer::Peer(void)
     , reservedConfiguration(nullptr)
     , authenticator(nullptr)
     , caAuthenticator(nullptr)
+    , configuredInputTensorInfo(nullptr)
 {
 }
 

diff --git a/subprojects/libbeyond-peer_nn/src/peer_grpc_client.cc b/subprojects/libbeyond-peer_nn/src/peer_grpc_client.cc
index 3c15d32175a6cd6b908631352bf4006d46925f3e..b984a73d1a785bc2fb6ede90fe44a6e6f143fea3 100644 (file)
--- a/subprojects/libbeyond-peer_nn/src/peer_grpc_client.cc
+++ b/subprojects/libbeyond-peer_nn/src/peer_grpc_client.cc
@@ -290,9 +290,7 @@ int Peer::GrpcClient::LoadModel(const char *modelFilename)
 
     // NOTE:
     // Reset cached model information
-    gst->GetModel()->SetInputTensorInfo(nullptr, 0);
-    gst->GetModel()->SetOutputTensorInfo(nullptr, 0);
-
+    gst->GetModel()->InvalidateTensorInfo();
     return static_cast<int>(response.status());
 }
 

diff --git a/subprojects/libbeyond-peer_nn/src/peer_model.cc b/subprojects/libbeyond-peer_nn/src/peer_model.cc
index 7391f82e2c1288f6fdc734c2d9aaf57bbafe8f0b..462d6dca7a422b70849d2c26dba001bcf177eede 100644 (file)
--- a/subprojects/libbeyond-peer_nn/src/peer_model.cc
+++ b/subprojects/libbeyond-peer_nn/src/peer_model.cc
@@ -75,6 +75,12 @@ int Peer::Model::GetOutputTensorInfo(const beyond_tensor_info *&info, int &size)
     return 0;
 }
 
+void Peer::Model::InvalidateTensorInfo(void)
+{
+    FreeTensorInfo(inputTensorInfo, inputTensorInfoSize);
+    FreeTensorInfo(outputTensorInfo, outputTensorInfoSize);
+}
+
 int Peer::Model::SetInputTensorInfo(beyond_tensor_info *info, int size)
 {
     FreeTensorInfo(inputTensorInfo, inputTensorInfoSize);