From: Sung-jae Park Date: Mon, 25 Oct 2021 07:59:59 +0000 (+0900) Subject: Update input_tensor_info for configuring the input X-Git-Tag: submit/tizen/20211028.105706~6 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=7a61d27816e7809b2b9232ff63bb91f821f4ca3f;p=platform%2Fcore%2Fml%2Fbeyond.git Update input_tensor_info for configuring the input [Problem] Input tensor information does not match to the configured input [Solution] Update the input_tensor_info by configured input information * SetInputTensorInfo() will change the model's input tensor information, even though the caller has configured the input. In other words, The GetInputTensorInfo() will not be affected by SetInputTensorInfo() if the input data is configured. * GetInputTensorInfo() only be affected by the configured input - SetInputTensorInfo() -> Always tries to change the input tensor specification of the model - SetOutputTensorInfo() -> Always tries to change the output tensor specification of the model - GetOutputTensorInfo() -> Always returns the output tensor specification of the model. - GetInputTensorInfo() -> returns the input tensor specification. If the input is configured, the configured input specification will be returned * Use the configuration input tensor info It is not necessary to get the model's input tensor info if there is a configured input information Change-Id: I313a0b9f3a5be4f8f0ae95b09be03ac76bc742d0 Signed-off-by: Sung-jae Park --- diff --git a/subprojects/libbeyond-peer_nn/include/peer.h b/subprojects/libbeyond-peer_nn/include/peer.h index 91182a2..1da653b 100644 --- a/subprojects/libbeyond-peer_nn/include/peer.h +++ b/subprojects/libbeyond-peer_nn/include/peer.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -94,6 +95,8 @@ private: char payload[1]; }; + static const std::map sizeMap; + private: Peer(void); virtual ~Peer(void); @@ -105,12 +108,15 @@ private: static void ConfigureImageInput(beyond_input_config *config, std::ostringstream &client_format, std::ostringstream &server_format); static void ConfigureVideoInput(beyond_input_config *config, std::ostringstream &client_format, std::ostringstream &server_format); + static beyond_tensor_info *AllocInputTensorInfo(beyond_input_config *config); static void ResetInfo(beyond_peer_info *&info); static void ResetRuntime(beyond_peer_info_runtime *&runtimes, int count_of_runtimes); static beyond_plugin_peer_nn_config *DuplicateConfig(const beyond_plugin_peer_nn_config *config); static void FreeConfig(beyond_plugin_peer_nn_config *&config); + static void FreeInputTensorInfo(beyond_tensor_info *&info); + private: std::unique_ptr serverCtx; std::unique_ptr clientCtx; @@ -121,6 +127,8 @@ private: beyond_plugin_peer_nn_config *reservedConfiguration; beyond::AuthenticatorInterface *authenticator; beyond::AuthenticatorInterface *caAuthenticator; + + beyond_tensor_info *configuredInputTensorInfo; }; #endif // __BEYOND_PEER_NN_PEER_H__ diff --git a/subprojects/libbeyond-peer_nn/include/peer_model.h b/subprojects/libbeyond-peer_nn/include/peer_model.h index 3ff1778..11a6a43 100644 --- a/subprojects/libbeyond-peer_nn/include/peer_model.h +++ b/subprojects/libbeyond-peer_nn/include/peer_model.h @@ -31,6 +31,7 @@ public: int GetOutputTensorInfo(const beyond_tensor_info *&info, int &size); int SetInputTensorInfo(beyond_tensor_info *info, int size); int SetOutputTensorInfo(beyond_tensor_info *info, int size); + void InvalidateTensorInfo(void); static void FreeTensorInfo(beyond_tensor_info *&info, int &size); static int DupTensorInfo(beyond_tensor_info *&dest, const beyond_tensor_info *src, int size); diff --git a/subprojects/libbeyond-peer_nn/src/peer.cc b/subprojects/libbeyond-peer_nn/src/peer.cc index bb35d70..2616e81 100644 --- a/subprojects/libbeyond-peer_nn/src/peer.cc +++ b/subprojects/libbeyond-peer_nn/src/peer.cc @@ -35,6 +35,81 @@ #include #define DEFAULT_FRAMEWORK "tensorflow-lite" +#define CONFIGURED_INPUT_TENSOR_SIZE 1 +#define CONFIGURED_INPUT_TENSOR_DIMS 2 + +// TODO: +// We have to update the sizeMap with more precise values for each of image format +const std::map Peer::sizeMap = { + { + "I420", // 640x480(Y), 320x240(U), 320x240(U) per a pixel + 3, // It is enough to represent YUV size in bytes + }, + { + "YV12", + 3, + }, + { + "YUY2", + 3, + }, + { + "UYVY", + 3, + }, + { + "Y41B", + 3, + }, + { + "Y42B", + 3, + }, + { + "YVYU", + 3, + }, + { + "Y444", + 3, + }, + { + "NV21", + 3, + }, + { + "NV12", + 3, + }, + { + "RGB", + 3, + }, + { + "BGR", + 3, + }, + { + "RGBx", + 4, + }, + { + "xRGB", + 4, + }, + { + "BGRx", + 4, + }, + { + "xBGR", + 4, + }, + { + "GRAY8", + 1, + } +}; Peer *Peer::Create(bool isServer, const char *framework, const char *accel, const char *storagePath) { @@ -122,6 +197,8 @@ void Peer::Destroy(void) eventObject = nullptr; ResetInfo(info); + FreeConfig(reservedConfiguration); + FreeInputTensorInfo(configuredInputTensorInfo); delete this; } @@ -205,6 +282,67 @@ void Peer::ConfigureVideoInput(beyond_input_config *config, std::ostringstream & server_format << " ! queue leaky=2 max-size-buffers=1"; } +beyond_tensor_info *Peer::AllocInputTensorInfo(beyond_input_config *config) +{ + const beyond_input_image_config *imageConfig = nullptr; + + if (config->input_type == BEYOND_INPUT_TYPE_IMAGE) { + imageConfig = &config->config.image; + } else if (config->input_type == BEYOND_INPUT_TYPE_VIDEO) { + imageConfig = &config->config.video.frame; + } else { + ErrPrint("Unknown input type configuration"); + return nullptr; + } + + if (imageConfig->format == nullptr) { + ErrPrint("Format is not specified"); + return nullptr; + } + + std::map::const_iterator it = sizeMap.find(imageConfig->format); + if (it == sizeMap.end()) { + ErrPrint("Unsupported format [%s]", imageConfig->format); + return nullptr; + } + + beyond_tensor_info *info = static_cast(malloc(sizeof(beyond_tensor_info) * CONFIGURED_INPUT_TENSOR_SIZE)); + if (info == nullptr) { + ErrPrintCode(errno, "malloc"); + return nullptr; + } + + info->size = imageConfig->width * imageConfig->height * it->second; + info->type = BEYOND_TENSOR_TYPE_UINT8; + info->name = strdup(imageConfig->format); + if (info->name == nullptr) { + ErrPrintCode(errno, "strdup"); + free(info); + return nullptr; + } + + // NOTE: + // WIDTH and HEIGHT, dimension size is always 2 + info->dims = static_cast(malloc(sizeof(beyond_tensor_info::dimensions) + sizeof(int) * CONFIGURED_INPUT_TENSOR_DIMS)); + if (info->dims == nullptr) { + ErrPrintCode(errno, "malloc"); + free(info->name); + info->name = nullptr; + free(info); + return nullptr; + } + + // TODO: + // This array information should be updated for each image format specification + // In case of the YUV, it could be manipulatd as a 3 dimensional array buffer + // At least now, image format related buffer layout will be treated as a 2-dimensional + // array (width and height) buffer + info->dims->size = CONFIGURED_INPUT_TENSOR_DIMS; + info->dims->data[0] = imageConfig->width; + info->dims->data[1] = imageConfig->height; + return info; +} + int Peer::ConfigureInput(const beyond_config *options) { if (serverCtx != nullptr) { @@ -279,6 +417,19 @@ int Peer::ConfigureInput(const beyond_config *options) return ret; } + beyond_tensor_info *newInfo = AllocInputTensorInfo(config); + if (newInfo == nullptr) { + int ret = -errno; + ErrPrintCode(errno, "strdup"); + free(client_desc); + client_desc = nullptr; + free(server_desc); + server_desc = nullptr; + return ret; + } + + FreeInputTensorInfo(configuredInputTensorInfo); + configuredInputTensorInfo = newInfo; _options->client.preprocessing = client_desc; _options->server.preprocessing = server_desc; } else { @@ -393,6 +544,15 @@ int Peer::GetInputTensorInfo(const beyond_tensor_info *&info, int &size) return -EILSEQ; } + // NOTE: + // If there is a configured input information, + // GetInputTensorInfo() should return the configured input tensor information + if (configuredInputTensorInfo) { + size = 1; + info = configuredInputTensorInfo; + return 0; + } + const beyond_tensor_info *_info = nullptr; int _size = 0; @@ -514,6 +674,20 @@ int Peer::AllocateTensor(const beyond_tensor_info *info, int size, beyond_tensor return 0; } +void Peer::FreeInputTensorInfo(beyond_tensor_info *&info) +{ + if (info == nullptr) { + return; + } + + free(info->name); + info->name = nullptr; + free(info->dims); + info->dims = nullptr; + free(info); + info = nullptr; +} + void Peer::FreeTensor(beyond_tensor *&tensor, int size) { if (clientCtx == nullptr) { @@ -1055,6 +1229,7 @@ Peer::Peer(void) , reservedConfiguration(nullptr) , authenticator(nullptr) , caAuthenticator(nullptr) + , configuredInputTensorInfo(nullptr) { } diff --git a/subprojects/libbeyond-peer_nn/src/peer_grpc_client.cc b/subprojects/libbeyond-peer_nn/src/peer_grpc_client.cc index 3c15d32..b984a73 100644 --- a/subprojects/libbeyond-peer_nn/src/peer_grpc_client.cc +++ b/subprojects/libbeyond-peer_nn/src/peer_grpc_client.cc @@ -290,9 +290,7 @@ int Peer::GrpcClient::LoadModel(const char *modelFilename) // NOTE: // Reset cached model information - gst->GetModel()->SetInputTensorInfo(nullptr, 0); - gst->GetModel()->SetOutputTensorInfo(nullptr, 0); - + gst->GetModel()->InvalidateTensorInfo(); return static_cast(response.status()); } diff --git a/subprojects/libbeyond-peer_nn/src/peer_model.cc b/subprojects/libbeyond-peer_nn/src/peer_model.cc index 7391f82..462d6dc 100644 --- a/subprojects/libbeyond-peer_nn/src/peer_model.cc +++ b/subprojects/libbeyond-peer_nn/src/peer_model.cc @@ -75,6 +75,12 @@ int Peer::Model::GetOutputTensorInfo(const beyond_tensor_info *&info, int &size) return 0; } +void Peer::Model::InvalidateTensorInfo(void) +{ + FreeTensorInfo(inputTensorInfo, inputTensorInfoSize); + FreeTensorInfo(outputTensorInfo, outputTensorInfoSize); +} + int Peer::Model::SetInputTensorInfo(beyond_tensor_info *info, int size) { FreeTensorInfo(inputTensorInfo, inputTensorInfoSize);