*/
typedef enum {
MV_INFERENCE_DATA_FLOAT32 = 0, /**< Data type of a given pre-trained model is float. */
- MV_INFERENCE_DATA_UINT8 /**< Data type of a given pre-trained model is unsigned char. */
+ MV_INFERENCE_DATA_UINT8, /**< Data type of a given pre-trained model is unsigned char. */
+ MV_INFERENCE_DATA_INT8 /**< Data type of a given pre-trained model is signed char. (Since 10.0) */
} mv_inference_data_type_e;
/**
int convertToCv(int given_type, int ch);
void colorConvert(cv::Mat &source, cv::Mat &dest, int sType, int dType);
void convertToCvSource(std::vector<mv_source_h> &mv_srcs, std::vector<cv::Mat> &cv_srcs);
- void normalize(cv::Mat &source, cv::Mat &dest, const std::vector<double> &mean, const std::vector<double> &std);
- void quantize(cv::Mat &source, cv::Mat &dest, const std::vector<double> &scale,
- const std::vector<double> &zeropoint);
+ void normalize(cv::Mat &source, cv::Mat &dest, bool signedInt8 = false);
+ void quantize(cv::Mat &source, cv::Mat &dest, bool signedInt8 = false);
};
} /* machine_learning */
{ "NHWC", INFERENCE_TENSOR_SHAPE_NHWC } };
std::map<std::string, mv_inference_data_type_e> gSupportedDataType = { { "FLOAT32", MV_INFERENCE_DATA_FLOAT32 },
- { "UINT8", MV_INFERENCE_DATA_UINT8 } };
+ { "UINT8", MV_INFERENCE_DATA_UINT8 },
+ { "INT8", MV_INFERENCE_DATA_INT8 } };
std::map<std::string, mv_colorspace_e> gSupportedColorType = { { "RGB888", MEDIA_VISION_COLORSPACE_RGB888 },
{ "GRAY8", MEDIA_VISION_COLORSPACE_Y800 } };
LOGI("LEAVE");
}
-void Preprocess::normalize(cv::Mat &source, cv::Mat &dest, const vector<double> &mean, const vector<double> &std)
+void Preprocess::normalize(cv::Mat &source, cv::Mat &dest, bool signedInt8)
{
LOGI("ENTER");
try {
- cv::subtract(source, cv::Scalar(mean[0], mean[1], mean[2]), dest);
- source = dest;
- cv::divide(source, cv::Scalar(std[0], std[1], std[2]), dest);
+ if (!signedInt8) {
+ cv::subtract(source, cv::Scalar(_config.mean[0], _config.mean[1], _config.mean[2]), dest);
+ source = dest;
+ cv::divide(source, cv::Scalar(_config.std[0], _config.std[1], _config.std[2]), dest);
+ } else {
+ std::vector<float> fData;
+
+ for (int idx = 0; idx < source.cols * source.rows; ++idx) {
+ fData.push_back(static_cast<float>(source.data[idx]) - _config.mean[0]);
+ dest.data[idx] = static_cast<char>(static_cast<int>(fData[idx]));
+ }
+ }
} catch (cv::Exception &e) {
throw InvalidOperation("Fail to substract/divide");
}
LOGI("LEAVE");
}
-void Preprocess::quantize(cv::Mat &source, cv::Mat &dest, const vector<double> &scale, const vector<double> &zeropoint)
+void Preprocess::quantize(cv::Mat &source, cv::Mat &dest, bool signedInt8)
{
LOGI("ENTER");
try {
- cv::subtract(source, cv::Scalar(zeropoint[0], zeropoint[1], zeropoint[2]), dest);
- source = dest;
- cv::multiply(source, cv::Scalar(scale[0], scale[1], scale[2]), dest);
+ if (!signedInt8) {
+ cv::subtract(source, cv::Scalar(_config.zeropoint[0], _config.zeropoint[1], _config.zeropoint[2]), dest);
+ source = dest;
+ cv::multiply(source, cv::Scalar(_config.scale[0], _config.scale[1], _config.scale[2]), dest);
+ } else {
+ std::vector<float> fData;
+
+ for (int idx = 0; idx < source.cols * source.rows; ++idx) {
+ fData.push_back(static_cast<float>(source.data[idx]) - _config.zeropoint[0]);
+ dest.data[idx] = fData[idx] * _config.scale[0];
+ }
+ }
} catch (cv::Exception &e) {
throw InvalidOperation("Fail to subtract/multiply");
}
int type = 0;
switch (given_type) {
- case INFERENCE_TENSOR_DATA_TYPE_UINT8:
case MV_INFERENCE_DATA_UINT8:
LOGI("Type is %d ch with UINT8", ch);
type = ch == 1 ? CV_8UC1 : CV_8UC3;
break;
- case INFERENCE_TENSOR_DATA_TYPE_FLOAT32:
+ case MV_INFERENCE_DATA_INT8:
+ LOGI("Type is %d ch with INT8", ch);
+ type = ch == 1 ? CV_8SC1 : CV_8SC3;
+ break;
case MV_INFERENCE_DATA_FLOAT32:
LOGI("Type is %d ch with FLOAT32", ch);
type = ch == 1 ? CV_32FC1 : CV_32FC3;
break;
default:
- LOGI("unknown data type so FLOAT32 data type will be used in default");
- type = ch == 1 ? CV_32FC1 : CV_32FC3;
- break;
+ LOGE("It's unknown data type(%d)", given_type);
+ throw InvalidParameter("Unknown data type.");
}
return type;
convertToCvSource(mv_srcs, oriCvSources);
inputVector.resize(_config.output_height * _config.output_width * _config.output_channel);
-
int data_type = convertToCv(_config.output_data_type, _config.output_channel);
+
// dest is a wrapper of the buffer.
cv::Mat dest(cv::Size(_config.output_width, _config.output_height), data_type, inputVector.data());
cv::Mat cvSource, cvDest;
// cvSource has new allocation with dest.size()
cv::resize(oriCvSources[0], cvSource, dest.size());
+ bool signedInt8DataType = (data_type == CV_8SC1 || data_type == CV_8SC3);
+
if (_config.skip_csc) {
cvSource.convertTo(dest, dest.type());
} else {
// cvDest is allocated if colorspace is not RGB888, and
// cvDest shares the data with cvSource if the colorspace is RGB888.
colorConvert(cvSource, cvDest, colorspace, _config.output_format);
- cvDest.convertTo(dest, dest.type());
+
+ // Convert data to a given data type only in case that the output data type isn't signed int8 type.
+ // If the output data type is signed int8 type, it should be converted after normalization
+ // because the normalized value can be negative. The range of signed int8 data type is -128 ~ 127
+ // so the pixel values more then 127 cannot be converted correctly.
+ if (!signedInt8DataType)
+ cvDest.convertTo(dest, dest.type());
}
- if (_config.normalize)
- normalize(dest, dest, _config.mean, _config.std);
+ if (_config.normalize) {
+ if (!signedInt8DataType)
+ normalize(dest, dest);
+ else
+ normalize(cvDest, dest, true);
+ }
- if (_config.quantize)
- quantize(dest, dest, _config.scale, _config.zeropoint);
+ if (_config.quantize) {
+ if (!signedInt8DataType || _config.normalize)
+ quantize(dest, dest);
+ else // If the output data type is signed int8 type then dest is empty and cvDest is used instead of dest.
+ quantize(cvDest, dest, true);
+ }
LOGI("LEAVE");
}
template void Preprocess::run<float>(mv_source_h &mv_src, vector<float> &inputVector);
template void Preprocess::run<unsigned char>(mv_source_h &mv_src, vector<unsigned char> &inputVector);
+template void Preprocess::run<char>(mv_source_h &mv_src, vector<char> &inputVector);
} /* machine_learning */
} /* mediavision */
#include <dlog.h>
#include "EngineConfig.h"
+#include "IGazeTracking.h"
#include "ITask.h"
#include "MvMlConfig.h"
-#include "IGazeTracking.h"
namespace mediavision
{
namespace machine_learning
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __TinyTracker_H__
+#define __TinyTracker_H__
+
+#include "mv_private.h"
+#include <memory>
+#include <mv_common.h>
+#include <string>
+
+#include "GazeTracking.h"
+#include <mv_inference_type.h>
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T> class TinyTracker : public GazeTracking<T>
+{
+ using GazeTracking<T>::_config;
+ using GazeTracking<T>::_preprocess;
+
+private:
+ GazeTrackingResult _result;
+
+public:
+ TinyTracker(GazeTrackingTaskType task_type, std::shared_ptr<Config> config);
+ ~TinyTracker();
+
+ GazeTrackingResult &result() override;
+};
+
+} // machine_learning
+} // mediavision
+
+#endif
\ No newline at end of file
enum class GazeTrackingTaskType {
GAZE_TRACKINGION_TASK_NONE = 0,
L2CS_NET,
- G_GAZE_ESTIMATION
+ G_GAZE_ESTIMATION,
+ TINY_TRACKER
// TODO
};
{
"name" : "MODEL_FILE_NAME",
"type" : "string",
- "value" : "generalizing_gaze_estimation_with_weak_supervision_from_synthetic_views_160x160_float16.tflite"
+ "value" : "TinyTrackerS.tflite"
},
{
"name" : "DEFAULT_MODEL_NAME",
"type" : "string",
- "value" : "G_GAZE_ESTIMATION"
+ "value" : "TINY_TRACKER"
},
{
"name" : "MODEL_META_FILE_NAME",
"type" : "string",
- "value" : "generalizing_gaze_estimation_with_weak_supervision_from_synthetic_views_160x160_float16.json"
+ "value" : "TinyTrackerS.json"
},
{
"name" : "BACKEND_TYPE",
template class GGazeEstimation<float>;
template class GGazeEstimation<unsigned char>;
+template class GGazeEstimation<char>;
}
}
template class GazeTracking<float>;
template class GazeTracking<unsigned char>;
+template class GazeTracking<char>;
}
}
*/
#include "GazeTrackingAdapter.h"
+#include "GGazeEstimation.h"
+#include "L2CSNet.h"
#include "MvMlException.h"
+#include "TinyTracker.h"
#include "gaze_tracking_type.h"
#include "mv_gaze_tracking_config.h"
-#include "L2CSNet.h"
-#include "GGazeEstimation.h"
using namespace std;
using namespace MediaVision::Common;
case GazeTrackingTaskType::G_GAZE_ESTIMATION:
_gaze_tracking = make_unique<GGazeEstimation<U> >(task_type, _config);
break;
+ case GazeTrackingTaskType::TINY_TRACKER:
+ _gaze_tracking = make_unique<TinyTracker<U> >(task_type, _config);
+ break;
default:
throw InvalidOperation("Invalid gaze tracking task type.");
}
case MV_INFERENCE_DATA_UINT8:
create<unsigned char>(task_type);
break;
+ case MV_INFERENCE_DATA_INT8:
+ create<char>(task_type);
+ break;
case MV_INFERENCE_DATA_FLOAT32:
create<float>(task_type);
break;
return GazeTrackingTaskType::L2CS_NET;
if (model_name == "G_GAZE_ESTIMATION")
return GazeTrackingTaskType::G_GAZE_ESTIMATION;
+ if (model_name == "TINY_TRACKER")
+ return GazeTrackingTaskType::TINY_TRACKER;
// TODO.
throw InvalidParameter("Invalid gaze tracking model name.");
template class L2CSNet<float>;
template class L2CSNet<unsigned char>;
-
+template class L2CSNet<char>;
}
}
--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <cmath>
+#include <map>
+#include <string.h>
+
+#include "MvMlException.h"
+#include "Postprocess.h"
+#include "TinyTracker.h"
+#include "mv_gaze_tracking_config.h"
+
+using namespace std;
+using namespace mediavision::inference;
+using namespace mediavision::machine_learning::exception;
+
+namespace mediavision
+{
+namespace machine_learning
+{
+template<typename T>
+TinyTracker<T>::TinyTracker(GazeTrackingTaskType task_type, std::shared_ptr<Config> config)
+ : GazeTracking<T>(task_type, config), _result()
+{}
+
+template<typename T> TinyTracker<T>::~TinyTracker()
+{}
+
+template<typename T> GazeTrackingResult &TinyTracker<T>::result()
+{
+ // Clear _result object because result() function can be called every time user wants
+ // so make sure to clear existing result data before getting the data again.
+ _result = GazeTrackingResult();
+
+ vector<string> names;
+
+ GazeTracking<T>::getOutputNames(names);
+
+ vector<float> outputTensor;
+
+ LOGD("names size = %zu", names.size());
+ for (unsigned int idx = 0; idx < names.size(); ++idx)
+ LOGD("%s", names[idx].c_str());
+
+ GazeTracking<T>::getOutputTensor(names[0], outputTensor);
+ float x = outputTensor[0];
+ float y = outputTensor[1];
+
+ LOGD("TinyTracker: x: %f, y: %f", x, y);
+
+ _result.frame_number++;
+ _result.number_of_faces = 1;
+ _result.x_pos.push_back(x * 10.0f);
+ _result.y_pos.push_back(y * 10.0f);
+ _result.yaws.push_back(0.0f);
+ _result.pitches.push_back(0.0f);
+
+ return _result;
+}
+
+template class TinyTracker<float>;
+template class TinyTracker<unsigned char>;
+template class TinyTracker<char>;
+
+}
+}
LOGI("Type is %d ch with UINT8", ch);
type = ch == 1 ? CV_8UC1 : CV_8UC3;
break;
+ case INFERENCE_TENSOR_DATA_TYPE_INT8:
+ LOGI("Type is %d ch with INT8", ch);
+ type = ch == 1 ? CV_8SC1 : CV_8SC3;
+ break;
case INFERENCE_TENSOR_DATA_TYPE_FLOAT32:
LOGI("Type is %d ch with FLOAT32", ch);
type = ch == 1 ? CV_32FC1 : CV_32FC3;
case MV_INFERENCE_DATA_UINT8:
type = INFERENCE_TENSOR_DATA_TYPE_UINT8;
break;
+ case MV_INFERENCE_DATA_INT8:
+ type = INFERENCE_TENSOR_DATA_TYPE_INT8;
+ break;
default:
LOGI("unknown data type so FLOAT32 data type will be used in default");
break;
template int Inference::run<float>(std::vector<std::vector<float> > &input_tensors);
template int Inference::run<unsigned char>(std::vector<std::vector<unsigned char> > &input_tensors);
+template int Inference::run<char>(std::vector<std::vector<char> > &input_tensors);
} /* Inference */
} /* MediaVision */
projects / platform / core / api / mediavision.git / commitdiff