[WIP. Tizen/API] Simple Single-Shot Inference Low-Level API

This is quite similar with CoreML/MLModel API, which is the
low-level API for iOS CoreML.

With this, application developers may invoke "inferences"
of a given model with a single data frame without
any pre/post-processings.

The corresponding pipeline will be:

app_src --> tensor_filter --> tensor_sink

with 0/1 framerate.

Signed-off-by: MyungJoo Ham <myungjoo.ham@samsung.com>

diff --git a/tizen-api/doc/nnstreamer_doc.h b/tizen-api/doc/nnstreamer_doc.h
index baa1dff..658ed23 100644 (file)
--- a/tizen-api/doc/nnstreamer_doc.h
+++ b/tizen-api/doc/nnstreamer_doc.h
@@ -76,6 +76,41 @@
  * <a href="https://developer.tizen.org/development/tizen-studio/native-tools/configuring-your-app/manifest-text-editor#feature">
  *    <b>Feature Element</b>.
  * </a>
+ *
+ * @defgroup CAPI_ML_NNSTREAMER_SINGLE_MODULE NNStreamer Single Shot
+ * @ingroup  CAPI_ML_NNSTREAMER
+ * @brief The NNStreamer Single API provides interfaces to invoke a neural network model with a single instance of input data.
+ * @section CAPI_ML_NNSTREAMER_SINGLE_HEADER Required Header
+ *   \#include <nnstreamer/nnstreamer-single.h> \n
+ *
+ * @section CAPI_ML_NNSTREAMER_SINGLE_OVERVIEW Overview
+ * The NNStreamer Single API provides interfaces to invoke a neural network model with a single instance of input data.
+ * This API is a syntactic sugar of NNStreamer Pipeline API with simplified features; thus, users are supposed to use NNStreamer Pipeline API directly if they want more advanced features.
+ * The user is expected to preprocess the input data for the given neural network model.
+ *
+ * This API allows the following operations with NNSTreamer:
+ * - Open a machine learning model (ml_model) with various mechanisms.
+ * - Close the model
+ * - Interfaces to enter a single instance of input data to the opened model.
+ * - Utility functions to handle opened model.
+ *
+ * Note that this API set is supposed to be thread-safe.
+ *
+ * @section CAPI_ML_NNSTREAMER_SINGLE_FEATURE Related Features
+ * This API is related with the following features:\n
+ *  - http://tizen.org/feature/nnstreamer.single\n
+ *
+ * It is recommended to probe feaqtures in your applicatoin for reliability.\n
+ * You can check if a device supports the related features for this API by using
+ * @ref CAPI_SYSTEM_SYSTEM_INFO_MODULE, thereby controlling the procedure of
+ * your application.\n
+ * To ensure your application is only running on the device with specific
+ * features, please define the features in your manifest file using the manifest
+ * editor in the SDK.\n
+ * More details on featuring your application can be found from
+ * <a href="https://developer.tizen.org/development/tizen-studio/native-tools/configuring-your-app/manifest-text-editor#feature">
+ *    <b>Feature Element</b>.
+ * </a>
  */
 
 

diff --git a/tizen-api/include/nnstreamer-single.h b/tizen-api/include/nnstreamer-single.h
new file mode 100644 (file)
index 0000000..ede3dc3
--- /dev/null
+++ b/tizen-api/include/nnstreamer-single.h
@@ -0,0 +1,241 @@
+/**
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file nnstreamer-single.h
+ * @date 29 March 2019
+ * @brief Tizen NNStreamer single shot invocation C-API Header.
+ *        This allows to invoke a neural network model directly.
+ * @see        https://github.com/nnsuite/nnstreamer
+ * @author MyungJoo Ham <myungjoo.ham@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ * @detail This is targetting Tizen 5.5 M2.
+ */
+
+#ifndef __TIZEN_NNSTREAMER_SINGLE_H__
+#define __TIZEN_NNSTREAMER_SINGLE_H__
+
+#include <stddef.h>
+#include <tizen_error.h>
+#include "nnstreamer.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+/**
+ * @addtogroup CAPI_ML_NNSTREAMER_SINGLE_MODULE
+ * @{
+ */
+
+/**
+ * @brief A handle of a simpleshot instance
+ * @since_tizen 5.5
+ */
+typedef void *ml_simpleshot_model_h;
+
+/* We can handle up to 16 tensors for an input or output frame */
+#define ML_MAX_TENSORS (16)
+/**
+ * @brief An instance of input or output frames. GstTensorsInfo is the metadata
+ * @since_tizen 5.5
+ */
+typedef struct {
+  void *tensor[ML_MAX_TENSORS]; /**< Tensor data. NULL for unused tensors. */
+  size_t size[ML_MAX_TENSORS]; /**< Size of each tensor. */
+  int num_tensors; /**< Number of tensors. > 0 && < ML_MAX_TENSORS. */
+} tensor_data;
+
+/**
+ * @brief Types of NNFWs.
+ * @since_tizen 5.5
+ */
+typedef enum {
+  ML_NNFW_UNKNOWN = 0, /**< it is unknown or we do not care this value. */
+  ML_NNFW_CUSTOM_FILTER, /**< custom filter (independent shared object). */
+  ML_NNFW_TENSORFLOW_LITE, /**< tensorflow-lite (.tflite). */
+  ML_NNFW_TENSORFLOW, /**< tensorflow (.pb). */
+} ml_model_nnfw;
+
+/**
+ * @brief Types of NNFWs. Note that if the affinity (nnn) is not supported by the driver or hardware, it is ignored.
+ * @since_tizen 5.5
+ */
+typedef enum {
+  ML_NNFW_HW_DO_NOT_CARE = 0, /**< Hardware resource is not specified. */
+  ML_NNFW_HW_AUTO = 1, /**< Try to schedule and optimize if possible. */
+  ML_NNFW_HW_CPU = 0x1000, /**< 0x1000: any CPU. 0x1nnn: CPU # nnn-1. */
+  ML_NNFW_HW_GPU = 0x2000, /**< 0x2000: any GPU. 0x2nnn: GPU # nnn-1. */
+  ML_NNFW_HW_NPU = 0x3000, /**< 0x3000: any NPU. 0x3nnn: NPU # nnn-1. */
+} ml_model_hw;
+
+/*************
+ * MAIN FUNC *
+ *************/
+/**
+ * @brief Open an ML model and return the model as a handle.
+ * @since_tizen 5.5
+ * @param[in] model_path This is the path to the neural network model file.
+ * @param[out] model This is the model opened. Users are required to close
+ *                   the given model with ml_model_close().
+ * @param[in] inputtype This is required if the given model has flexible input
+ *                      dimension, where the input dimension MUST be given
+ *                      before executing the model.
+ *                      However, once it's given, the input dimension cannot
+ *                      be changed for the given model handle.
+ *                      Is is required by some custom filters of nnstreamer.
+ *                      You may set NULL if it's not required.
+ * @param[in] nnfw The nerual network framework used to open the given
+ *                 @model_path. Set ML_NNFW_UNKNOWN to let it auto-detect.
+ * @param[in] hw Tell the corresponding @nnfw to use a specific hardware.
+ *               Set ML_NNFW_HW_DO_NOT_CARE if it does not matter.
+ * @return @c 0 on success. otherwise a negative error value
+ * @retval #NNS_ERROR_NONE Successful
+ *
+ * @detail Even if the model has flexible input data dimensions,
+ *         input data frames of an instance of a model should share the
+ *         same dimension.
+ */
+int ml_model_open (const char *model_path, ml_simpleshot_model_h *model,
+    const nns_tensors_info_s *inputtype, ml_model_nnfw nnfw,
+    ml_model_hw hw);
+
+/**
+ * @brief Close the opened model handle.
+ * @since_tizen 5.5
+ * @param[in] model The model handle to be closed.
+ * @return @c 0 on success. otherwise a negative error value
+ * @retval #NNS_ERROR_NONE Successful
+ * @retval #NNS_ERROR_INVALID_PARAMETER Fail. The parameter is invalid (pipe is NULL?)
+ */
+int ml_model_close (ml_simpleshot_model_h model);
+
+/**
+ * @brief Invoke the model with the given input data.
+ * @since_tizen 5.5
+ * @param[in] model The model to be inferred.
+ * @param[in] input The input data to be inferred.
+ * @param[out] output The output buffer. Set NULL if you want to let
+ *                    this function to allocate a new output buffer.
+ * @return @c The output buffer. If @output is NULL, this is a newly
+ *         allocated buffer; thus, the user needs to free it.
+ *         If there is an error, this is set NULL. Check get_last_result()
+ *         of tizen_error.h in such cases.
+ *
+ * @detail Even if the model has flexible input data dimensions,
+ *         input data frames of an instance of a model should share the
+ *         same dimension.
+ */
+tensor_data * ml_model_inference (ml_simpleshot_model_h model,
+    const tensor_data *input, tensor_data *output);
+
+/*************
+ * UTILITIES *
+ *************/
+
+/**
+ * @brief Get type (tensor dimension, type, name and so on) of required input
+ *        data for the given model.
+ * @detail Note that a model may not have such
+ *         information if its input type is flexible.
+ *         Besides, names of tensors may be not available while dimensions and
+ *         types are available.
+ * @since_tizen 5.5
+ * @param[in] model The model to be investigated
+ * @param[out] inputtype The type of input tensor.
+ * @return @c 0 on success. otherwise a negative error value
+ * @retval #NNS_ERROR_NONE Successful
+ */
+int ml_model_get_inputtype (ml_simpleshot_model_h model,
+    nns_tensors_info_s *inputtype);
+
+/**
+ * @brief Get type (tensor dimension, type, name and so on) of output
+ *        data of the given model.
+ * @detail Note that a model may not have such
+ *         information if its input type is flexible and output type is
+ *         not determined statically.
+ *         Besides, names of tensors may be not available while dimensions and
+ *         types are available.
+ * @since_tizen 5.5
+ * @param[in] model The model to be investigated
+ * @param[out] outputtype The type of output tensor.
+ * @return @c 0 on success. otherwise a negative error value
+ * @retval #NNS_ERROR_NONE Successful
+ */
+int ml_model_get_outputtype (ml_simpleshot_model_h model,
+    nns_tensors_info_s *outputtype);
+
+/**
+ * @brief Get the byte size of the given tensor type.
+ * @since_tizen 5.5
+ * @param[in] tensor_type The tensor type to be investigated.
+ * @return @c >= 0 on success with byte size. otherwise a negative error value
+ */
+int ml_model_get_tensor_size (const nns_tensor_info_s *tensor_type);
+
+/**
+ * @brief Get the byte size of the given tensors type.
+ * @since_tizen 5.5
+ * @param[in] tensors_type The tensors type to be investigated.
+ * @return @c >= 0 on success with byte size. otherwise a negative error value
+ */
+int ml_model_get_tensors_size (const nns_tensors_info_s *tensors_type);
+
+/**
+ * @brief Free the tensors type pointer
+ * @since_tizen 5.5
+ * @param[in] type the tensors type pointer to be freed.
+ */
+void ml_model_free_tensorsinfo (nns_tensors_info_s *type);
+
+/**
+ * @brief Free the tensors data pointer
+ * @since_tizen 5.5
+ * @param[in] tensor the tensors data pointer to be freed.
+ */
+void ml_model_free_tensordata (tensor_data *tensor);
+
+/**
+ * @brief Allocate a tensor data frame with the given tensors type.
+ * @since_tizen 5.5
+ * @param[in] type the tensors type pointer for the allocation
+ * @return @c Tensors data pointer allocated. Null if error.
+ * @retval NULL there is an error. call get_last_result() to get specific
+ *         error numbers.
+ */
+tensor_data *ml_model_allocate_tensors (const nns_tensors_info_s *type);
+
+/**
+ * @brief Check the availability of the given execution environments.
+ * @since_tizen 5.5
+ * @param[in] nnfw Check if the nnfw is available in the system.
+ *                 Set ML_NNFW_UNKNOWN to skip checking nnfw.
+ * @param[in] hw Check if the hardware is available in the system.
+ *               Set ML_NNFW_HW_DO_NOT_CARE to skip checking hardware.
+ * @return @c 0 if it's available. 1 if it's not available.
+ *            negative value if there is an error.
+ * @retval #NNS_ERROR_NONE Successful and the environments are available.
+ * @retval 1 Successful but the environments are not available.
+ */
+int ml_model_check_nnfw (ml_model_nnfw nnfw, ml_model_hw hw);
+
+/**
+ * @}
+ */
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+#endif /* __TIZEN_NNSTREAMER_SINGLE_H__ */

diff --git a/tizen-api/src/nnstreamer-single.c b/tizen-api/src/nnstreamer-single.c
new file mode 100644 (file)
index 0000000..76f1ea7
--- /dev/null
+++ b/tizen-api/src/nnstreamer-single.c
@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file nnstreamer-simple.c
+ * @date 08 May 2019
+ * @brief Tizen NNStreamer/Simple C-API Wrapper.
+ *        This allows to invoke individual input frame with NNStreamer.
+ * @see        https://github.com/nnsuite/nnstreamer
+ * @author MyungJoo Ham <myungjoo.ham@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+
+#include <nnstreamer.h>         /* Uses NNStreamer/Pipeline C-API */
+
+typedef struct
+{
+  nns_pipeline_h pipe;
+} ml_simpleshot_model;
+/**
+ * ml_simpleshot_model *model = g_new0 (ml_simpleshot_model, 1);
+ * ml_simpleshot_model_h *model_h;
+ * *model_h = model;
+ */
+
+/**
+ * @brief Refer to nnstreamer-single.h
+ */
+int
+ml_model_open (const char *model_path, ml_simpleshot_model_h * model,
+    const nns_tensors_info_s * inputtype, ml_model_nnfw nnfw, ml_model_hw hw)
+{
+  ml_simpleshot_model_h *_model;
+  int ret = NNS_ERROR_NONE;
+  char *pipedesc;               /* pipeline description */
+
+  /* 1. Determine nnfw */
+
+  /* 2. Determine hw */
+
+  /* 3. Determine input dimension ==> caps_filter string */
+
+  /* 4. Construct a pipeline */
+  _model = g_new (ml_simpleshot_model, 1);
+  ret = nns_pipeline_construct (pipedesc, &_model->pipe);
+
+  /* 5. Allocate */
+  *model = _model;
+}