--- /dev/null
+/**
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __SHARED_BUFFER_H__
+#define __SHARED_BUFFER_H__
+
+#include <atomic>
+#include <vector>
+#include <memory>
+
+#include "SingleoCommonTypes.h"
+
+namespace singleo
+{
+class SharedBuffer
+{
+private:
+ std::vector<std::shared_ptr<BaseDataType> > _data;
+ std::atomic<int> ref_count;
+
+public:
+ SharedBuffer() = default;
+ ~SharedBuffer() = default;
+
+ void addInput(std::shared_ptr<BaseDataType> data)
+ {
+ _data.push_back(data);
+ }
+
+ std::vector<std::shared_ptr<BaseDataType> > &getInputs()
+ {
+ return _data;
+ }
+
+ void addRef()
+ {
+ ref_count.fetch_add(1, std::memory_order_relaxed);
+ }
+
+ void release()
+ {
+ if (ref_count.fetch_sub(1, std::memory_order_acq_rel) == 1)
+ for (auto &data : _data)
+ data->release();
+ }
+};
+
+}
+
+#endif
\ No newline at end of file
struct BaseDataType {
DataType _data_type { DataType::NONE };
+ bool is_owned { false };
+
BaseDataType(DataType data_type) : _data_type(data_type)
{}
virtual ~BaseDataType()
{}
virtual std::shared_ptr<BaseDataType> clone() = 0;
+ virtual void release() = 0;
};
struct FileDataType : public BaseDataType {
{
return std::make_shared<FileDataType>(*this);
}
+
+ void release() override
+ {}
};
struct ImageDataType : public BaseDataType {
unsigned int height {};
unsigned int byte_per_pixel {};
ImagePixelFormat pixel_format { ImagePixelFormat::NONE };
- bool custom { false };
ImageDataType() : BaseDataType(DataType::IMAGE)
{}
- ImageDataType(const ImageDataType &other) : BaseDataType(DataType::IMAGE)
+ std::shared_ptr<BaseDataType> clone() override
{
- unsigned int size = other.width * other.height * other.byte_per_pixel;
+ auto newImage = std::make_shared<ImageDataType>();
+
+ unsigned int size = width * height * byte_per_pixel;
if (size == 0) {
SINGLEO_LOGE("ImageDataType::operator=() : Invalid size");
throw singleo::exception::InvalidOperation("ImageDataType::operator=() : Invalid size");
}
- ptr = new unsigned char[size];
- memcpy(ptr, other.ptr, size);
+ newImage->ptr = new unsigned char[size];
+ memcpy(newImage->ptr, this->ptr, size);
- width = other.width;
- height = other.height;
- byte_per_pixel = other.byte_per_pixel;
- pixel_format = other.pixel_format;
- }
+ newImage->width = width;
+ newImage->height = height;
+ newImage->byte_per_pixel = byte_per_pixel;
+ newImage->pixel_format = pixel_format;
+ newImage->is_owned = true;
- ImageDataType &operator=(const ImageDataType &other) = delete;
+ return newImage;
+ }
- std::shared_ptr<BaseDataType> clone() override
+ void release() override
{
- return std::make_shared<ImageDataType>(*this);
+ if (is_owned) {
+ delete[] ptr;
+ ptr = nullptr;
+ }
}
};
{
return std::make_shared<RawDataType>(*this);
}
+
+ void release() override
+ {
+ if (is_owned) {
+ delete[] ptr;
+ ptr = nullptr;
+ }
+ }
};
enum class ResultType { NONE, OBJECT_DETECTION, FACE_DETECTION, FACE_LANDMARK };
{
ImagePreprocessor preprocessor(data);
ImageDataType preprocessed = dynamic_cast<ImageDataType &>(preprocessor.getData());
- ImageDataType copied(preprocessed);
+ auto copied = dynamic_pointer_cast<ImageDataType>(preprocessed.clone());
+
+ // Set is_owned to false to prevent from releasing copied image data
+ // because the owner of copied is AutoZoom so the copied will be released
+ // in performAsync function.
+ // Ps. if is_owned is true, copied image data can be released while task manager is being performed.
+ copied->is_owned = false;
if (_user_cb) {
// if postprocessor isn't in progress, postprocess current camera preview image.
if (!_postprocessor->isWorking()) {
- ImageDataType zoom_data = dynamic_cast<ImageDataType &>(_postprocessor->getOutput(copied));
+ ImageDataType zoom_data = dynamic_cast<ImageDataType &>(_postprocessor->getOutput(*copied));
_user_cb(zoom_data.ptr, zoom_data.width, zoom_data.height, zoom_data.byte_per_pixel, _user_data);
} else {
- _user_cb(copied.ptr, copied.width, copied.height, copied.byte_per_pixel, _user_data);
+ _user_cb(copied->ptr, copied->width, copied->height, copied->byte_per_pixel, _user_data);
}
}
// Make sure to release copied buffer if incoming queue isn't empty so skipped pushing the buffer.
// If empty then push the buffer to the incoming queue. This buffer will be released at the end of runTaskManager function.
- if (_async_manager->pushInput(copied) != SINGLEO_ERROR_NONE)
- delete copied.ptr;
+ if (_async_manager->pushInput(*copied) != SINGLEO_ERROR_NONE)
+ delete copied->ptr;
}
bool AutoZoom::isKeyValid(std::string key)
auto imageData = dynamic_cast<ImageDataType &>(data);
- // A given imageData has been used so release it here.
- delete imageData.ptr;
+ // A given imageData - allocated by inputFeedCb function - has been used so release it here.
+ if (!imageData.is_owned)
+ delete imageData.ptr;
});
}
std::copy(d->results().begin(), d->results().end(), std::back_inserter(_results));
_cb(this);
+
+ _inputBuffer->release();
}
};
#include "INode.h"
#include "SingleoException.h"
+#include "SharedBuffer.h"
namespace singleo
{
std::string _name;
std::vector<std::shared_ptr<INode> > _dependencies;
NodeCb _cb;
- std::vector<std::shared_ptr<BaseDataType> > _inputs;
- std::shared_ptr<BaseDataType> _output;
+ std::shared_ptr<SharedBuffer> _inputBuffer;
+ std::shared_ptr<SharedBuffer> _outputBuffer;
std::vector<std::shared_ptr<BaseResultType> > _results;
bool _completed { false };
std::condition_variable _event;
{
return _name;
}
- void addInput(std::shared_ptr<BaseDataType> input) override;
- std::vector<std::shared_ptr<BaseDataType> > &getInputs() override;
+ void setInputBuffer(std::shared_ptr<SharedBuffer> inputBuffer) override;
+ std::shared_ptr<SharedBuffer> &getInputBuffer() override;
void addDependency(std::shared_ptr<INode> node) override;
std::vector<std::shared_ptr<INode> > &getDependencies() override;
- void setOutput(std::shared_ptr<BaseDataType> output) override;
- std::shared_ptr<BaseDataType> &getOutput() override;
+ void setOutputBuffer(std::shared_ptr<SharedBuffer> outputBuffer) override;
+ std::shared_ptr<SharedBuffer> &getOutputBuffer() override;
void wait() override;
void wakeup() override;
virtual void configure() = 0;
if (_cb)
_cb(this);
+
+ _inputBuffer->release();
}
};
#include "SingleoCommonTypes.h"
#include "ServiceDataType.h"
+#include "SharedBuffer.h"
namespace singleo
{
virtual NodeType getType() = 0;
virtual std::string &getName() = 0;
- virtual void addInput(std::shared_ptr<BaseDataType> input) = 0;
- virtual std::vector<std::shared_ptr<BaseDataType> > &getInputs() = 0;
+ virtual void setInputBuffer(std::shared_ptr<SharedBuffer> inputBuffer) = 0;
+ virtual std::shared_ptr<SharedBuffer> &getInputBuffer() = 0;
virtual void addDependency(std::shared_ptr<INode> node) = 0;
virtual std::vector<std::shared_ptr<INode> > &getDependencies() = 0;
- virtual void setOutput(std::shared_ptr<BaseDataType> output) = 0;
- virtual std::shared_ptr<BaseDataType> &getOutput() = 0;
+ virtual void setOutputBuffer(std::shared_ptr<SharedBuffer> outputBuffer) = 0;
+ virtual std::shared_ptr<SharedBuffer> &getOutputBuffer() = 0;
virtual void configure() = 0;
virtual void invoke() = 0;
virtual std::vector<std::shared_ptr<BaseResultType> > &results() = 0;
#include "INode.h"
#include "SingleoException.h"
+#include "SharedBuffer.h"
namespace singleo
{
NodeType _type { NodeType::NONE };
std::string _name;
std::vector<std::shared_ptr<INode> > _dependencies;
- std::vector<std::shared_ptr<BaseDataType> > _inputs;
- std::shared_ptr<BaseDataType> _output;
+ std::shared_ptr<SharedBuffer> _inputBuffer;
+ std::shared_ptr<SharedBuffer> _outputBuffer;
bool _completed { false };
std::condition_variable _event;
std::mutex _mutex;
{
return _name;
}
- void addInput(std::shared_ptr<BaseDataType> input) override;
- std::vector<std::shared_ptr<BaseDataType> > &getInputs() override;
+ void setInputBuffer(std::shared_ptr<SharedBuffer> inputBuffer) override;
+ std::shared_ptr<SharedBuffer> &getInputBuffer() override;
void addDependency(std::shared_ptr<INode> node) override;
std::vector<std::shared_ptr<INode> > &getDependencies() override;
- void setOutput(std::shared_ptr<BaseDataType> output) override;
- std::shared_ptr<BaseDataType> &getOutput() override;
+ void setOutputBuffer(std::shared_ptr<SharedBuffer> outputBuffer) override;
+ std::shared_ptr<SharedBuffer> &getOutputBuffer() override;
void wait() override;
void wakeup() override;
virtual void configure() = 0;
return _type;
}
-void CallbackNode::addInput(shared_ptr<BaseDataType> input)
+void CallbackNode::setInputBuffer(shared_ptr<SharedBuffer> inputBuffer)
{
- _inputs.push_back(input);
+ inputBuffer->addRef();
+ _inputBuffer = inputBuffer;
}
-vector<shared_ptr<BaseDataType> > &CallbackNode::getInputs()
+shared_ptr<SharedBuffer> &CallbackNode::getInputBuffer()
{
- return _inputs;
+ return _inputBuffer;
}
void CallbackNode::setCb(const NodeCb &cb)
return _dependencies;
}
-void CallbackNode::setOutput(std::shared_ptr<BaseDataType> output)
+void CallbackNode::setOutputBuffer(std::shared_ptr<SharedBuffer> outputBuffer)
{
- _output = output;
+ outputBuffer->addRef();
+ _outputBuffer = outputBuffer;
}
-std::shared_ptr<BaseDataType> &CallbackNode::getOutput()
+std::shared_ptr<SharedBuffer> &CallbackNode::getOutputBuffer()
{
- return _output;
+ return _outputBuffer;
}
std::vector<std::shared_ptr<BaseResultType> > &CallbackNode::results()
void InferenceNode::invoke()
{
- // TODO. consider for multiple inputs later.
- _task->invoke(*_inputs[0]);
+ auto &inputs = _inputBuffer->getInputs();
- auto input = dynamic_pointer_cast<ImageDataType>(_inputs[0]);
+ // TODO. consider for multiple inputs later.
+ _task->invoke(*inputs[0]);
- // Inference request has been completed so release input data if the data was internally allocated
- // by callback node.
- if (input->custom)
- delete input->ptr;
+ // Inference request has been completed so release input data.
+ _inputBuffer->release();
- _inputs.clear();
_results.clear();
_resultMutex.lock();
_results.push_back(_task->result().clone());
+
_resultMutex.unlock();
}
if (!node->getDependencies().empty()) {
// TODO. consider for multiple dependencies later.
auto &callbackNode = node->getDependencies()[0];
+ auto &outputBuffer = callbackNode->getOutputBuffer();
- node->addInput(callbackNode->getOutput());
+ node->setInputBuffer(outputBuffer);
+
+ // output buffer has been shared to node so release it here.
+ outputBuffer->release();
}
} else {
// TODO. consider for mulitple inputs later.
- node->addInput(_inputs[0]);
+ auto inputBuffer = make_shared<SharedBuffer>();
+ for (auto &i : _inputs)
+ inputBuffer->addInput(i);
+
+ node->setInputBuffer(inputBuffer);
}
node->invoke();
void TaskManager::addInput(BaseDataType &input)
{
+ auto new_input = make_shared<ImageDataType>();
auto imageData = dynamic_cast<ImageDataType &>(input);
- _inputs.push_back(make_shared<ImageDataType>(imageData));
+ *new_input = imageData;
+ _inputs.push_back(new_input);
}
vector<shared_ptr<BaseDataType> > &TaskManager::getInputs()
std::vector<std::unique_ptr<std::thread> > threads;
+ auto inputBuffer = make_shared<SharedBuffer>();
+
+ for (auto &i : _inputs)
+ inputBuffer->addInput(i);
+
for (auto &n : _nodes) {
// Set input as source of current node.
// If no dependency then this node has to receive input source
}
// TODO. consider for multiple sources later.
- n->addInput(_inputs[0]);
+ n->setInputBuffer(inputBuffer);
}
// Wait until all nodes added to this node as dependency are completed
return _type;
}
-void TaskNode::addInput(shared_ptr<BaseDataType> input)
+void TaskNode::setInputBuffer(shared_ptr<SharedBuffer> inputBuffer)
{
- _inputs.push_back(input);
+ inputBuffer->addRef();
+ _inputBuffer = inputBuffer;
}
-vector<shared_ptr<BaseDataType> > &TaskNode::getInputs()
+shared_ptr<SharedBuffer> &TaskNode::getInputBuffer()
{
- return _inputs;
+ return _inputBuffer;
}
void TaskNode::addDependency(std::shared_ptr<INode> node)
return _dependencies;
}
-void TaskNode::setOutput(std::shared_ptr<BaseDataType> output)
+void TaskNode::setOutputBuffer(std::shared_ptr<SharedBuffer> outputBuffer)
{
- _output = output;
+ outputBuffer->addRef();
+ _outputBuffer = outputBuffer;
}
-std::shared_ptr<BaseDataType> &TaskNode::getOutput()
+std::shared_ptr<SharedBuffer> &TaskNode::getOutputBuffer()
{
- return _output;
+ return _outputBuffer;
}
void TaskNode::wait()
void BridgeNodeCallback(INode *node)
{
auto callbackNode = dynamic_cast<CallbackNode *>(node);
- shared_ptr<ImageDataType> imageData = dynamic_pointer_cast<ImageDataType>(callbackNode->getInputs()[0]);
- ImageDataType newImage(*imageData);
+ auto &inputBuffer = callbackNode->getInputBuffer();
+ auto newBaseData = inputBuffer->getInputs()[0]->clone();
+ auto newImage = dynamic_pointer_cast<ImageDataType>(newBaseData);
const int answer[][4] = { { 553, 87, 583, 129 }, { 397, 110, 427, 149 } };
- newImage.custom = true;
-
- cv::Mat cv_image(cv::Size(newImage.width, newImage.height), CV_MAKETYPE(CV_8U, 3), newImage.ptr);
+ cv::Mat cv_image(cv::Size(newImage->width, newImage->height), CV_MAKETYPE(CV_8U, 3), newImage->ptr);
auto &results = callbackNode->results();
for (auto r : results) {
}
}
- callbackNode->setOutput(make_shared<ImageDataType>(newImage));
+ auto outputBuffer = make_shared<SharedBuffer>();
+ outputBuffer->addInput(newBaseData);
+ callbackNode->setOutputBuffer(outputBuffer);
}
// GraphA : input ----> face_detection ----> bridge ----> face_landmark_detection ----> endpoint ----> output
// -----> bridge -----> face_landmark_detection --------
// input -----> face_detection --| |----> endpoint ----> output
// -----------------------------------------------------
-TEST(SingloTaskManager, MultipleNodesBasedGraphDShouldWork)
+TEST(SingloTaskManager, MultipleNodesBasedGraphCShouldWork)
{
cv::Mat cv_image = cv::imread(IMG_FACE, cv::IMREAD_COLOR);
cv::cvtColor(cv_image, cv_image, cv::COLOR_BGR2RGB);
taskManager->clear();
}
}
+
+// GraphD:
+// -----> face_landmark_detection --------
+// input -----> face_detection ------> bridge --| |----> endpoint ----> output
+// -----> face_detection -----------------
+TEST(SingloTaskManager, MultipleNodesBasedGraphDShouldWork)
+{
+ cv::Mat cv_image = cv::imread(IMG_FACE, cv::IMREAD_COLOR);
+ cv::cvtColor(cv_image, cv_image, cv::COLOR_BGR2RGB);
+ const int fd_answer[][4] = { { 553, 87, 583, 129 }, { 397, 110, 427, 149 } };
+ const int fld_answer[][2] = { { 243, 133 }, { 404, 135 }, { 347, 197 }, { 250, 266 }, { 385, 266 } };
+
+ ASSERT_FALSE(cv_image.empty());
+
+ ImageDataType image_data;
+
+ image_data.width = cv_image.cols;
+ image_data.height = cv_image.rows;
+ image_data.byte_per_pixel = cv_image.channels();
+ image_data.ptr = cv_image.data;
+
+ auto factory = InferenceTaskFactory::instance().create("MvInferenceTaskFactory");
+
+ auto taskManager = make_unique<TaskManager>();
+ const unsigned int maxIteration = 10;
+
+ for (unsigned int cnt = 0; cnt < maxIteration; ++cnt) {
+ taskManager->addInput(image_data);
+
+ auto face_detection_node_a = make_shared<InferenceNode>("face_detection_a");
+ face_detection_node_a->setInferenceTask(factory->createFaceDetection());
+ taskManager->addNode(face_detection_node_a);
+
+ auto bridge_node = make_shared<BridgeNode>();
+ bridge_node->setCb(BridgeNodeCallback);
+ bridge_node->addDependency(face_detection_node_a);
+ taskManager->addNode(bridge_node);
+
+ auto face_landmark_node = make_shared<InferenceNode>("face_landmark");
+ face_landmark_node->setInferenceTask(factory->createFaceLandmarkDetection());
+ face_landmark_node->addDependency(bridge_node);
+ taskManager->addNode(face_landmark_node);
+
+ auto face_detection_node_b = make_shared<InferenceNode>("face_detection_b");
+ face_detection_node_b->setInferenceTask(factory->createFaceDetection());
+ face_detection_node_b->addDependency(bridge_node);
+ taskManager->addNode(face_detection_node_b);
+
+ auto endpoint_node = make_shared<EndpointNode>();
+ endpoint_node->setCb(LastNodeCallback);
+ endpoint_node->addDependency(face_detection_node_b);
+ endpoint_node->addDependency(face_landmark_node);
+ taskManager->addNode(endpoint_node);
+
+ taskManager->run();
+
+ auto results = taskManager->output();
+ for (auto &result : results) {
+ if (result->_type == ResultType::FACE_DETECTION) {
+ auto fd_result = dynamic_pointer_cast<FdResultType>(result);
+ unsigned int idx = 0;
+
+ for (auto rect : fd_result->_rects) {
+ ASSERT_EQ(rect.left, fd_answer[idx][0]);
+ ASSERT_EQ(rect.top, fd_answer[idx][1]);
+ ASSERT_EQ(rect.right, fd_answer[idx][2]);
+ ASSERT_EQ(rect.bottom, fd_answer[idx++][3]);
+ }
+ } else {
+ auto fld_result = dynamic_pointer_cast<FldResultType>(result);
+ unsigned int idx = 0;
+
+ for (auto &point : fld_result->_points) {
+ ASSERT_EQ(point.x, fld_answer[idx][0]);
+ ASSERT_EQ(point.y, fld_answer[idx++][1]);
+ }
+ }
+ }
+
+ taskManager->clear();
+ }
+}
projects / platform / core / api / singleo.git / commitdiff