input/camera_api: add sync API support

Add sync API support by implementing capture method of CameraApiBackend class.

According to Camera-api document below,
https://docs.tizen.org/application/native/api/mobile/4.0/group__CAPI__MEDIA__CAMERA__MODULE.html
For capture, this patch follows the Camera API state diagram. However,
one tricky way is needed to make sure to wait for the "Captured" state
because completed callback since camera_start_capture call will be called
by main thread so sync API of singleo framework cannot use the callback.
Therefore, to make sure to wait for the state, we use a tricky way by
checking if current state is "Captured".

Change-Id: Ibe2ba23e0c1c9bb215976949a7d1a05cd1f98db6
Signed-off-by: Inki Dae <inki.dae@samsung.com>

diff --git a/input/backends/camera_api/include/CameraApiBackend.h b/input/backends/camera_api/include/CameraApiBackend.h
index 21ffbc7921e3b11d89f4b1b25c4bb797c0b2ee5d..20982cae618c9343578d19a0e9e7acbd41d5f849 100644 (file)
--- a/input/backends/camera_api/include/CameraApiBackend.h
+++ b/input/backends/camera_api/include/CameraApiBackend.h
@@ -42,10 +42,16 @@ private:
        std::unique_ptr<std::thread> _thread_handle;
        bool _exit_thread { false };
        std::mutex _preview_mutex;
+       std::mutex _capture_mutex;
        std::condition_variable _preview_event;
-       camera_pixel_format_e _defaultPixelFormat { CAMERA_PIXEL_FORMAT_I420 };
-       std::vector<camera_pixel_format_e> _validPixelFormat;
-       std::vector<cv::Size> _validResolution;
+       std::condition_variable _capture_event;
+       cv::Mat _cvCaptureImage;
+       camera_pixel_format_e _defaultPreviewPixelFormat { CAMERA_PIXEL_FORMAT_I420 };
+       camera_pixel_format_e _defaultCapturePixelFormat { CAMERA_PIXEL_FORMAT_I420 };
+       std::vector<camera_pixel_format_e> _validPreviewPixelFormat;
+       std::vector<camera_pixel_format_e> _validCapturePixelFormat;
+       std::vector<cv::Size> _validPreviewResolution;
+       std::vector<cv::Size> _validCaptureResolution;
        std::unordered_set<unsigned int> _camera_ids;
        camera_device_e _active_camera_id {};
        std::map<int, camera_device_e> _cameraDeviceIdTable = {
@@ -54,13 +60,28 @@ private:
                { 6, CAMERA_DEVICE_CAMERA6 }, { 7, CAMERA_DEVICE_CAMERA7 }, { 8, CAMERA_DEVICE_CAMERA8 },
                { 9, CAMERA_DEVICE_CAMERA9 }
        };
+       std::map<camera_pixel_format_e, std::string> _pixelFormatTable = {
+               { CAMERA_PIXEL_FORMAT_NV12, "NV12" },     { CAMERA_PIXEL_FORMAT_NV12T, "NV12T" },
+               { CAMERA_PIXEL_FORMAT_NV16, "NV16" },     { CAMERA_PIXEL_FORMAT_NV21, "NV21" },
+               { CAMERA_PIXEL_FORMAT_YUYV, "YUYV" },     { CAMERA_PIXEL_FORMAT_UYVY, "UYVY" },
+               { CAMERA_PIXEL_FORMAT_422P, "422P" },     { CAMERA_PIXEL_FORMAT_I420, "I420" },
+               { CAMERA_PIXEL_FORMAT_YV12, "YV12" },     { CAMERA_PIXEL_FORMAT_RGB565, "RGB565" },
+               { CAMERA_PIXEL_FORMAT_RGB888, "RGB888" }, { CAMERA_PIXEL_FORMAT_RGBA, "RGBA" },
+               { CAMERA_PIXEL_FORMAT_ARGB, "ARGB" },     { CAMERA_PIXEL_FORMAT_JPEG, "JPEG" },
+               { CAMERA_PIXEL_FORMAT_H264, "H264" },     { CAMERA_PIXEL_FORMAT_INVZ, "INVZ" }
+       };
 
        void updateAvailableCameraDevices();
        void setActivateCameraDevice(unsigned int id);
        void threadLoop();
        static void previewCb(camera_preview_data_s *data, void *user_data);
+       static void captureCb(camera_image_data_s *image, camera_image_data_s *postview, camera_image_data_s *thumbnail,
+                                                 void *user_data);
        static bool previewFormatCb(camera_pixel_format_e format, void *user_data);
+       static bool captureFormatCb(camera_pixel_format_e format, void *user_data);
        static bool previewResolutionCb(int width, int height, void *user_data);
+       static bool captureResolutionCb(int width, int height, void *user_data);
+       static void captureCompletedCb(void *user_data);
        static bool compareSizesDescending(const cv::Size &a, const cv::Size &b);
 
 public:

diff --git a/input/backends/camera_api/src/CameraApiBackend.cpp b/input/backends/camera_api/src/CameraApiBackend.cpp
index 7eb03af33611ea4c4bb07e2711434aa2428f7d38..8b7caaf2eb00e84608adf873dc4845ff7607465b 100644 (file)
--- a/input/backends/camera_api/src/CameraApiBackend.cpp
+++ b/input/backends/camera_api/src/CameraApiBackend.cpp
@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+#include <chrono>
+
 #include "SingleoException.h"
 #include "CameraApiBackend.h"
 #include "SingleoLog.h"
@@ -115,19 +117,18 @@ bool CameraApiBackend::previewFormatCb(camera_pixel_format_e format, void *user_
 {
        CameraApiBackend *context = static_cast<CameraApiBackend *>(user_data);
 
-       static map<camera_pixel_format_e, string> pixel_format_table = {
-               { CAMERA_PIXEL_FORMAT_NV12, "NV12" },     { CAMERA_PIXEL_FORMAT_NV12T, "NV12T" },
-               { CAMERA_PIXEL_FORMAT_NV16, "NV16" },     { CAMERA_PIXEL_FORMAT_NV21, "NV21" },
-               { CAMERA_PIXEL_FORMAT_YUYV, "YUYV" },     { CAMERA_PIXEL_FORMAT_UYVY, "UYVY" },
-               { CAMERA_PIXEL_FORMAT_422P, "422P" },     { CAMERA_PIXEL_FORMAT_I420, "I420" },
-               { CAMERA_PIXEL_FORMAT_YV12, "YV12" },     { CAMERA_PIXEL_FORMAT_RGB565, "RGB565" },
-               { CAMERA_PIXEL_FORMAT_RGB888, "RGB888" }, { CAMERA_PIXEL_FORMAT_RGBA, "RGBA" },
-               { CAMERA_PIXEL_FORMAT_ARGB, "ARGB" },     { CAMERA_PIXEL_FORMAT_JPEG, "JPEG" },
-               { CAMERA_PIXEL_FORMAT_H264, "H264" },     { CAMERA_PIXEL_FORMAT_INVZ, "INVZ" }
-       };
+       SINGLEO_LOGD("supported pixel format for preview = %s", context->_pixelFormatTable[format].c_str());
+       context->_validPreviewPixelFormat.push_back(format);
+
+       return true;
+}
+
+bool CameraApiBackend::captureFormatCb(camera_pixel_format_e format, void *user_data)
+{
+       CameraApiBackend *context = static_cast<CameraApiBackend *>(user_data);
 
-       SINGLEO_LOGD("supported pixel format = %s", pixel_format_table[format].c_str());
-       context->_validPixelFormat.push_back(format);
+       SINGLEO_LOGD("supported pixel format for capture = %s", context->_pixelFormatTable[format].c_str());
+       context->_validCapturePixelFormat.push_back(format);
 
        return true;
 }
@@ -141,10 +142,24 @@ bool CameraApiBackend::previewResolutionCb(int width, int height, void *user_dat
 {
        auto context = static_cast<CameraApiBackend *>(user_data);
 
-       SINGLEO_LOGD("supported resolution : width = %d, height = %d", width, height);
+       SINGLEO_LOGD("supported resolution for preview: width = %d, height = %d", width, height);
 
-       context->_validResolution.push_back(cv::Size(width, height));
-       sort(context->_validResolution.begin(), context->_validResolution.end(), context->compareSizesDescending);
+       context->_validPreviewResolution.push_back(cv::Size(width, height));
+       sort(context->_validPreviewResolution.begin(), context->_validPreviewResolution.end(),
+                context->compareSizesDescending);
+
+       return true;
+}
+
+bool CameraApiBackend::captureResolutionCb(int width, int height, void *user_data)
+{
+       auto context = static_cast<CameraApiBackend *>(user_data);
+
+       SINGLEO_LOGD("supported resolution for capture: width = %d, height = %d", width, height);
+
+       context->_validCaptureResolution.push_back(cv::Size(width, height));
+       sort(context->_validCaptureResolution.begin(), context->_validCaptureResolution.end(),
+                context->compareSizesDescending);
 
        return true;
 }
@@ -157,16 +172,31 @@ void CameraApiBackend::configure()
                throw InvalidOperation("CameraApiBackend: camera_foreach_supported_preview_resolution failed.");
        }
 
-       SINGLEO_LOGD("Set camera resulition with width(%d) and height(%d) in default.", _validResolution[0].width,
-                                _validResolution[0].height);
+       ret = camera_foreach_supported_capture_resolution(_camera, captureResolutionCb, this);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_foreach_supported_capture_resolution failed. ret: %d", ret);
+               throw InvalidOperation("CameraApiBackend: camera_foreach_supported_capture_resolution failed.");
+       }
+
+       SINGLEO_LOGD("Set camera resulition for preview with width(%d) and height(%d) in default.",
+                                _validPreviewResolution[0].width, _validPreviewResolution[0].height);
 
        // TODO. set user-given resolution with the option string of singleo API later.
-       ret = camera_set_preview_resolution(_camera, _validResolution[0].width, _validResolution[0].height);
+       ret = camera_set_preview_resolution(_camera, _validPreviewResolution[0].width, _validPreviewResolution[0].height);
        if (ret != CAMERA_ERROR_NONE) {
                SINGLEO_LOGE("CameraApiBackend: camera_set_preview_resolution failed. ret: %d", ret);
                throw InvalidOperation("CameraApiBackend: camera_set_preview_resolution failed.");
        }
 
+       SINGLEO_LOGD("Set camera resulition for capture with width(%d) and height(%d) in default.",
+                                _validCaptureResolution[0].width, _validCaptureResolution[0].height);
+
+       ret = camera_set_capture_resolution(_camera, _validCaptureResolution[0].width, _validCaptureResolution[0].height);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_set_capture_resolution failed. ret: %d", ret);
+               throw InvalidOperation("CameraApiBackend: camera_set_capture_resolution failed.");
+       }
+
        ret = camera_foreach_supported_preview_format(_camera, previewFormatCb, this);
        if (ret != CAMERA_ERROR_NONE) {
                SINGLEO_LOGE("CameraApiBackend: camera_foreach_supported_preview_format failed. ret: %d", ret);
@@ -174,38 +204,130 @@ void CameraApiBackend::configure()
        }
 
        // TODO. read default pixel format from json configuration file later.
-       auto it = find(_validPixelFormat.begin(), _validPixelFormat.end(), _defaultPixelFormat);
-       if (it == _validPixelFormat.end()) {
-               SINGLEO_LOGE("Invalid pixel format type.");
-               throw InvalidOperation("CameraApiBackend: Invalid pixel format type.");
+       auto preview_it =
+                       find(_validPreviewPixelFormat.begin(), _validPreviewPixelFormat.end(), _defaultPreviewPixelFormat);
+       if (preview_it == _validPreviewPixelFormat.end()) {
+               SINGLEO_LOGE("Invalid preview pixel format type.");
+               throw InvalidOperation("CameraApiBackend: Invalid preview pixel format type.");
        }
 
-       ret = camera_set_preview_format(_camera, _defaultPixelFormat);
+       ret = camera_set_preview_format(_camera, _defaultPreviewPixelFormat);
        if (ret != CAMERA_ERROR_NONE) {
                SINGLEO_LOGE("CameraApiBackend: camera_set_preview_format failed. ret: %d", ret);
                throw InvalidOperation("CameraApiBackend: camera_set_preview_format failed.");
        }
 
-       ret = camera_set_preview_cb(_camera, previewCb, this);
+       ret = camera_foreach_supported_capture_format(_camera, captureFormatCb, this);
        if (ret != CAMERA_ERROR_NONE) {
-               SINGLEO_LOGE("CameraApiBackend: camera_set_preview_cb failed. ret: %d", ret);
-               throw InvalidOperation("CameraApiBackend: camera_set_preview_cb failed.");
+               SINGLEO_LOGE("CameraApiBackend: camera_foreach_supported_capture_format failed. ret: %d", ret);
+               throw InvalidOperation("CameraApiBackend: camera_foreach_supported_capture_format failed.");
        }
+
+       // TODO. read default pixel format from json configuration file later.
+       auto capture_it =
+                       find(_validCapturePixelFormat.begin(), _validCapturePixelFormat.end(), _defaultCapturePixelFormat);
+       if (capture_it == _validCapturePixelFormat.end()) {
+               SINGLEO_LOGE("Invalid capture pixel format type.");
+               throw InvalidOperation("CameraApiBackend: Invalid capture pixel format type.");
+       }
+
+       ret = camera_set_capture_format(_camera, _defaultCapturePixelFormat);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_set_capture_format failed. ret: %d", ret);
+               throw InvalidOperation("CameraApiBackend: camera_set_capture_format failed.");
+       }
+}
+
+void CameraApiBackend::captureCb(camera_image_data_s *image, camera_image_data_s *postview,
+                                                                camera_image_data_s *thumbnail, void *user_data)
+{
+       auto context = static_cast<CameraApiBackend *>(user_data);
+
+       if (image->format != CAMERA_PIXEL_FORMAT_I420) {
+               SINGLEO_LOGE("Pixel format not supported.");
+               throw InvalidParameter("Pixel format not supported.");
+       }
+
+       // TODO. consider for other pixel format support later.
+
+       cv::Mat cv_src(image->height + image->height / 2, image->width, CV_8UC1, image->data);
+
+       cv::cvtColor(cv_src, context->_cvCaptureImage, cv::COLOR_YUV2BGR_I420);
+
+       context->_capture_event.notify_one();
+}
+
+void CameraApiBackend::captureCompletedCb(void *user_data)
+{
+       // Nothing to do
 }
 
 void CameraApiBackend::capture(BaseDataType &out_data)
 {
-       throw InvalidOperation("Not supported yet");
+       // Capture sequence should "start preview -> start capture ->  start preview -> stop preview"
+       // according to the Camera API state digram below,
+       // https://docs.tizen.org/application/native/api/mobile/4.0/group__CAPI__MEDIA__CAMERA__MODULE.html
+
+       // Change the state to "Previewing" from "Created".
+       int ret = camera_start_preview(_camera);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_start_preview failed. ret: %d", ret);
+               throw InvalidOperation("CameraApiBackend: camera_start_preview failed.");
+       }
+
+       // Change the state to "Capturing" and then "Captured".
+       // Ps. captureCb callback will be called by another thread context camera-api internally
+       //     but by main thread for captureCompletedCb callback.
+       ret = camera_start_capture(_camera, captureCb, captureCompletedCb, this);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_start_capture failed. ret: %d", ret);
+               throw InvalidOperation("camera_start_preview: camera_start_capture failed.");
+       }
+
+       unique_lock<mutex> lock(_capture_mutex);
+
+       _capture_event.wait(lock);
+
+       auto &image_data = dynamic_cast<ImageDataType &>(out_data);
+
+       image_data.width = _cvCaptureImage.cols;
+       image_data.height = _cvCaptureImage.rows;
+       image_data.pixel_format = ImagePixelFormat::RGB888;
+       image_data.byte_per_pixel = _cvCaptureImage.channels();
+       image_data.ptr = _cvCaptureImage.data;
+
+       // Wait for "Captured" state.
+       // Ps. captureCompletedCb callback is called by main thread so we cannot use the callback
+       //     to wait for "Captured" state with sync API.
+       camera_state_e state {};
+
+       camera_get_state(_camera, &state);
+       while (state != CAMERA_STATE_CAPTURED) {
+               this_thread::sleep_for(10ms);
+               camera_get_state(_camera, &state);
+       }
+
+       // Change the state to "Previewing".
+       ret = camera_start_preview(_camera);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_start_preview failed. ret: %d", ret);
+               throw InvalidOperation("CameraApiBackend: camera_start_preview failed.");
+       }
+
+       // Change the state to "Created".
+       ret = camera_stop_preview(_camera);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_stop_preview failed. ret: %d", ret);
+               throw InvalidOperation("CameraApiBackend: camera_stop_preview failed.");
+       }
 }
 
 void CameraApiBackend::threadLoop()
 {
-       SINGLEO_LOGD("CameraApiBackend: stream off.");
-
        int ret = camera_start_preview(_camera);
        if (ret != CAMERA_ERROR_NONE) {
                SINGLEO_LOGE("CameraApiBackend: camera_start_preview failed. ret: %d", ret);
-               throw InvalidOperation("camera_start_preview: camera_set_preview_format failed.");
+               throw InvalidOperation("CameraApiBackend: camera_start_preview failed.");
        }
 
        unique_lock<mutex> lock(_preview_mutex);
@@ -216,6 +338,12 @@ void CameraApiBackend::streamOn()
 {
        SINGLEO_LOGD("CameraApiBackend: stream on.");
 
+       int ret = camera_set_preview_cb(_camera, previewCb, this);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_set_preview_cb failed. ret: %d", ret);
+               throw InvalidOperation("CameraApiBackend: camera_set_preview_cb failed.");
+       }
+
        if (!_thread_handle)
                _thread_handle = std::make_unique<std::thread>(&CameraApiBackend::threadLoop, this);
 }
@@ -230,6 +358,12 @@ void CameraApiBackend::streamOff()
                throw InvalidOperation("camera_start_preview: camera_stop_preview failed.");
        }
 
+       ret = camera_unset_preview_cb(_camera);
+       if (ret != CAMERA_ERROR_NONE) {
+               SINGLEO_LOGE("CameraApiBackend: camera_unset_preview_cb failed. ret: %d", ret);
+               throw InvalidOperation("camera_start_preview: camera_unset_preview_cb failed.");
+       }
+
        _exit_thread = true;
        _preview_event.notify_one();
        _thread_handle->join();

diff --git a/services/auto_zoom/src/AutoZoom.cpp b/services/auto_zoom/src/AutoZoom.cpp
index b599d5eb0b399c63f80517ec483e0428614db2c7..40e21fe23338522aae48ea6e8f13b8a6881d62e6 100644 (file)
--- a/services/auto_zoom/src/AutoZoom.cpp
+++ b/services/auto_zoom/src/AutoZoom.cpp
@@ -202,6 +202,7 @@ void AutoZoom::updateResult(BaseDataType &in_data)
        SINGLEO_LOGD("detected object count = %zu", autozoom_result.num_of_objects);
 
        if (autozoom_result.num_of_objects == 0) {
+               _result = autozoom_result;
                SINGLEO_LOGW("No detected objects.");
                return;
        }
@@ -219,7 +220,6 @@ void AutoZoom::getResultCnt(unsigned int *cnt)
                _postprocessor->addInput(_result);
        }
 
-       // TODO. Temparary code.
        *cnt = static_cast<unsigned int>(_result.num_of_objects);
 }
 

diff --git a/test/services/test_autozoom.cpp b/test/services/test_autozoom.cpp
index 1167e9b245c19fedd5344f82b5ad33847768c97f..8176d45882a51695a72b012b0879afe3d7a627f3 100644 (file)
--- a/test/services/test_autozoom.cpp
+++ b/test/services/test_autozoom.cpp
@@ -73,27 +73,31 @@ TEST(AutoZoomTest, InferenceRequestWithCameraInputFeedShouldBeOk)
        int ret = singleo_service_create("service=auto_zoom, input_feed=camera, camera_id=0, fps=30, async=0", &handle);
        ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
 
-       ret = singleo_service_perform(handle);
-       ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
-
-       unsigned int cnt;
+       const unsigned int max_iteration = 10;
 
-       ret = singleo_service_get_result_cnt(handle, &cnt);
-       ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+       for (unsigned int repeat_cnt = 0; repeat_cnt < max_iteration; ++repeat_cnt) {
+               ret = singleo_service_perform(handle);
+               ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
 
-       for (unsigned int idx = 0; idx < cnt; ++idx) {
-               unsigned int x, y, w, h;
+               unsigned int cnt;
 
-               ret = singleo_service_get_result_int(handle, idx, "x", &x);
-               ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
-               ret = singleo_service_get_result_int(handle, idx, "y", &y);
-               ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
-               ret = singleo_service_get_result_int(handle, idx, "width", &w);
-               ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
-               ret = singleo_service_get_result_int(handle, idx, "height", &h);
+               ret = singleo_service_get_result_cnt(handle, &cnt);
                ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
 
-               cout << x << " x " << y << " ~ " << w << " x " << h << endl;
+               for (unsigned int idx = 0; idx < cnt; ++idx) {
+                       unsigned int x, y, w, h;
+
+                       ret = singleo_service_get_result_int(handle, idx, "x", &x);
+                       ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+                       ret = singleo_service_get_result_int(handle, idx, "y", &y);
+                       ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+                       ret = singleo_service_get_result_int(handle, idx, "width", &w);
+                       ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+                       ret = singleo_service_get_result_int(handle, idx, "height", &h);
+                       ASSERT_EQ(ret, SINGLEO_ERROR_NONE);
+
+                       cout << x << " x " << y << " ~ " << w << " x " << h << endl;
+               }
        }
 
        ret = singleo_service_destroy(handle);