Imported Upstream version 6.1

[platform/upstream/ffmpeg.git] / libavfilter / dnn_interface.h

diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index d3a0c58..183d841 100644 (file)
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -30,15 +30,16 @@
 #include "libavutil/frame.h"
 #include "avfilter.h"
 
-typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
+#define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')
 
-typedef enum {DNN_NATIVE, DNN_TF, DNN_OV} DNNBackendType;
+typedef enum {DNN_TF = 1, DNN_OV} DNNBackendType;
 
 typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
 
 typedef enum {
     DCO_NONE,
     DCO_BGR,
+    DCO_RGB,
 } DNNColorOrder;
 
 typedef enum {
@@ -52,17 +53,43 @@ typedef enum {
     DFT_NONE,
     DFT_PROCESS_FRAME,      // process the whole frame
     DFT_ANALYTICS_DETECT,   // detect from the whole frame
-    // we can add more such as detect_from_crop, classify_from_bbox, etc.
+    DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
 }DNNFunctionType;
 
+typedef enum {
+    DL_NONE,
+    DL_NCHW,
+    DL_NHWC,
+} DNNLayout;
+
 typedef struct DNNData{
     void *data;
     int width, height, channels;
     // dt and order together decide the color format
     DNNDataType dt;
     DNNColorOrder order;
+    DNNLayout layout;
+    float scale;
+    float mean;
 } DNNData;
 
+typedef struct DNNExecBaseParams {
+    const char *input_name;
+    const char **output_names;
+    uint32_t nb_output;
+    AVFrame *in_frame;
+    AVFrame *out_frame;
+} DNNExecBaseParams;
+
+typedef struct DNNExecClassificationParams {
+    DNNExecBaseParams base;
+    const char *target;
+} DNNExecClassificationParams;
+
+typedef int (*FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx);
+typedef int (*DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx);
+typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);
+
 typedef struct DNNModel{
     // Stores model that can be different for different backends.
     void *model;
@@ -74,37 +101,37 @@ typedef struct DNNModel{
     DNNFunctionType func_type;
     // Gets model input information
     // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
-    DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name);
+    int (*get_input)(void *model, DNNData *input, const char *input_name);
     // Gets model output width/height with given input w/h
-    DNNReturnType (*get_output)(void *model, const char *input_name, int input_width, int input_height,
+    int (*get_output)(void *model, const char *input_name, int input_width, int input_height,
                                 const char *output_name, int *output_width, int *output_height);
     // set the pre process to transfer data from AVFrame to DNNData
     // the default implementation within DNN is used if it is not provided by the filter
-    int (*pre_proc)(AVFrame *frame_in, DNNData *model_input, AVFilterContext *filter_ctx);
+    FramePrePostProc frame_pre_proc;
     // set the post process to transfer data from DNNData to AVFrame
     // the default implementation within DNN is used if it is not provided by the filter
-    int (*post_proc)(AVFrame *frame_out, DNNData *model_output, AVFilterContext *filter_ctx);
+    FramePrePostProc frame_post_proc;
+    // set the post process to interpret detect result from DNNData
+    DetectPostProc detect_post_proc;
+    // set the post process to interpret classify result from DNNData
+    ClassifyPostProc classify_post_proc;
 } DNNModel;
 
 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
 typedef struct DNNModule{
     // Loads model and parameters from given file. Returns NULL if it is not possible.
     DNNModel *(*load_model)(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
-    // Executes model with specified input and output. Returns DNN_ERROR otherwise.
-    DNNReturnType (*execute_model)(const DNNModel *model, const char *input_name, AVFrame *in_frame,
-                                   const char **output_names, uint32_t nb_output, AVFrame *out_frame);
-    // Executes model with specified input and output asynchronously. Returns DNN_ERROR otherwise.
-    DNNReturnType (*execute_model_async)(const DNNModel *model, const char *input_name, AVFrame *in_frame,
-                                         const char **output_names, uint32_t nb_output, AVFrame *out_frame);
+    // Executes model with specified input and output. Returns the error code otherwise.
+    int (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params);
     // Retrieve inference result.
-    DNNAsyncStatusType (*get_async_result)(const DNNModel *model, AVFrame **in, AVFrame **out);
+    DNNAsyncStatusType (*get_result)(const DNNModel *model, AVFrame **in, AVFrame **out);
     // Flush all the pending tasks.
-    DNNReturnType (*flush)(const DNNModel *model);
+    int (*flush)(const DNNModel *model);
     // Frees memory allocated for model.
     void (*free_model)(DNNModel **model);
 } DNNModule;
 
 // Initializes DNNModule depending on chosen backend.
-DNNModule *ff_get_dnn_module(DNNBackendType backend_type);
+const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);
 
 #endif