From: Inki Dae Date: Thu, 18 Apr 2024 01:21:38 +0000 (+0900) Subject: input/camera_api: add sync API support X-Git-Tag: accepted/tizen/unified/20240903.110722~67 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F92%2F309892%2F3;p=platform%2Fcore%2Fapi%2Fsingleo.git input/camera_api: add sync API support Add sync API support by implementing capture method of CameraApiBackend class. According to Camera-api document below, https://docs.tizen.org/application/native/api/mobile/4.0/group__CAPI__MEDIA__CAMERA__MODULE.html For capture, this patch follows the Camera API state diagram. However, one tricky way is needed to make sure to wait for the "Captured" state because completed callback since camera_start_capture call will be called by main thread so sync API of singleo framework cannot use the callback. Therefore, to make sure to wait for the state, we use a tricky way by checking if current state is "Captured". Change-Id: Ibe2ba23e0c1c9bb215976949a7d1a05cd1f98db6 Signed-off-by: Inki Dae --- diff --git a/input/backends/camera_api/include/CameraApiBackend.h b/input/backends/camera_api/include/CameraApiBackend.h index 21ffbc7..20982ca 100644 --- a/input/backends/camera_api/include/CameraApiBackend.h +++ b/input/backends/camera_api/include/CameraApiBackend.h @@ -42,10 +42,16 @@ private: std::unique_ptr _thread_handle; bool _exit_thread { false }; std::mutex _preview_mutex; + std::mutex _capture_mutex; std::condition_variable _preview_event; - camera_pixel_format_e _defaultPixelFormat { CAMERA_PIXEL_FORMAT_I420 }; - std::vector _validPixelFormat; - std::vector _validResolution; + std::condition_variable _capture_event; + cv::Mat _cvCaptureImage; + camera_pixel_format_e _defaultPreviewPixelFormat { CAMERA_PIXEL_FORMAT_I420 }; + camera_pixel_format_e _defaultCapturePixelFormat { CAMERA_PIXEL_FORMAT_I420 }; + std::vector _validPreviewPixelFormat; + std::vector _validCapturePixelFormat; + std::vector _validPreviewResolution; + std::vector _validCaptureResolution; std::unordered_set _camera_ids; camera_device_e _active_camera_id {}; std::map _cameraDeviceIdTable = { @@ -54,13 +60,28 @@ private: { 6, CAMERA_DEVICE_CAMERA6 }, { 7, CAMERA_DEVICE_CAMERA7 }, { 8, CAMERA_DEVICE_CAMERA8 }, { 9, CAMERA_DEVICE_CAMERA9 } }; + std::map _pixelFormatTable = { + { CAMERA_PIXEL_FORMAT_NV12, "NV12" }, { CAMERA_PIXEL_FORMAT_NV12T, "NV12T" }, + { CAMERA_PIXEL_FORMAT_NV16, "NV16" }, { CAMERA_PIXEL_FORMAT_NV21, "NV21" }, + { CAMERA_PIXEL_FORMAT_YUYV, "YUYV" }, { CAMERA_PIXEL_FORMAT_UYVY, "UYVY" }, + { CAMERA_PIXEL_FORMAT_422P, "422P" }, { CAMERA_PIXEL_FORMAT_I420, "I420" }, + { CAMERA_PIXEL_FORMAT_YV12, "YV12" }, { CAMERA_PIXEL_FORMAT_RGB565, "RGB565" }, + { CAMERA_PIXEL_FORMAT_RGB888, "RGB888" }, { CAMERA_PIXEL_FORMAT_RGBA, "RGBA" }, + { CAMERA_PIXEL_FORMAT_ARGB, "ARGB" }, { CAMERA_PIXEL_FORMAT_JPEG, "JPEG" }, + { CAMERA_PIXEL_FORMAT_H264, "H264" }, { CAMERA_PIXEL_FORMAT_INVZ, "INVZ" } + }; void updateAvailableCameraDevices(); void setActivateCameraDevice(unsigned int id); void threadLoop(); static void previewCb(camera_preview_data_s *data, void *user_data); + static void captureCb(camera_image_data_s *image, camera_image_data_s *postview, camera_image_data_s *thumbnail, + void *user_data); static bool previewFormatCb(camera_pixel_format_e format, void *user_data); + static bool captureFormatCb(camera_pixel_format_e format, void *user_data); static bool previewResolutionCb(int width, int height, void *user_data); + static bool captureResolutionCb(int width, int height, void *user_data); + static void captureCompletedCb(void *user_data); static bool compareSizesDescending(const cv::Size &a, const cv::Size &b); public: diff --git a/input/backends/camera_api/src/CameraApiBackend.cpp b/input/backends/camera_api/src/CameraApiBackend.cpp index 7eb03af..8b7caaf 100644 --- a/input/backends/camera_api/src/CameraApiBackend.cpp +++ b/input/backends/camera_api/src/CameraApiBackend.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include + #include "SingleoException.h" #include "CameraApiBackend.h" #include "SingleoLog.h" @@ -115,19 +117,18 @@ bool CameraApiBackend::previewFormatCb(camera_pixel_format_e format, void *user_ { CameraApiBackend *context = static_cast(user_data); - static map pixel_format_table = { - { CAMERA_PIXEL_FORMAT_NV12, "NV12" }, { CAMERA_PIXEL_FORMAT_NV12T, "NV12T" }, - { CAMERA_PIXEL_FORMAT_NV16, "NV16" }, { CAMERA_PIXEL_FORMAT_NV21, "NV21" }, - { CAMERA_PIXEL_FORMAT_YUYV, "YUYV" }, { CAMERA_PIXEL_FORMAT_UYVY, "UYVY" }, - { CAMERA_PIXEL_FORMAT_422P, "422P" }, { CAMERA_PIXEL_FORMAT_I420, "I420" }, - { CAMERA_PIXEL_FORMAT_YV12, "YV12" }, { CAMERA_PIXEL_FORMAT_RGB565, "RGB565" }, - { CAMERA_PIXEL_FORMAT_RGB888, "RGB888" }, { CAMERA_PIXEL_FORMAT_RGBA, "RGBA" }, - { CAMERA_PIXEL_FORMAT_ARGB, "ARGB" }, { CAMERA_PIXEL_FORMAT_JPEG, "JPEG" }, - { CAMERA_PIXEL_FORMAT_H264, "H264" }, { CAMERA_PIXEL_FORMAT_INVZ, "INVZ" } - }; + SINGLEO_LOGD("supported pixel format for preview = %s", context->_pixelFormatTable[format].c_str()); + context->_validPreviewPixelFormat.push_back(format); + + return true; +} + +bool CameraApiBackend::captureFormatCb(camera_pixel_format_e format, void *user_data) +{ + CameraApiBackend *context = static_cast(user_data); - SINGLEO_LOGD("supported pixel format = %s", pixel_format_table[format].c_str()); - context->_validPixelFormat.push_back(format); + SINGLEO_LOGD("supported pixel format for capture = %s", context->_pixelFormatTable[format].c_str()); + context->_validCapturePixelFormat.push_back(format); return true; } @@ -141,10 +142,24 @@ bool CameraApiBackend::previewResolutionCb(int width, int height, void *user_dat { auto context = static_cast(user_data); - SINGLEO_LOGD("supported resolution : width = %d, height = %d", width, height); + SINGLEO_LOGD("supported resolution for preview: width = %d, height = %d", width, height); - context->_validResolution.push_back(cv::Size(width, height)); - sort(context->_validResolution.begin(), context->_validResolution.end(), context->compareSizesDescending); + context->_validPreviewResolution.push_back(cv::Size(width, height)); + sort(context->_validPreviewResolution.begin(), context->_validPreviewResolution.end(), + context->compareSizesDescending); + + return true; +} + +bool CameraApiBackend::captureResolutionCb(int width, int height, void *user_data) +{ + auto context = static_cast(user_data); + + SINGLEO_LOGD("supported resolution for capture: width = %d, height = %d", width, height); + + context->_validCaptureResolution.push_back(cv::Size(width, height)); + sort(context->_validCaptureResolution.begin(), context->_validCaptureResolution.end(), + context->compareSizesDescending); return true; } @@ -157,16 +172,31 @@ void CameraApiBackend::configure() throw InvalidOperation("CameraApiBackend: camera_foreach_supported_preview_resolution failed."); } - SINGLEO_LOGD("Set camera resulition with width(%d) and height(%d) in default.", _validResolution[0].width, - _validResolution[0].height); + ret = camera_foreach_supported_capture_resolution(_camera, captureResolutionCb, this); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_foreach_supported_capture_resolution failed. ret: %d", ret); + throw InvalidOperation("CameraApiBackend: camera_foreach_supported_capture_resolution failed."); + } + + SINGLEO_LOGD("Set camera resulition for preview with width(%d) and height(%d) in default.", + _validPreviewResolution[0].width, _validPreviewResolution[0].height); // TODO. set user-given resolution with the option string of singleo API later. - ret = camera_set_preview_resolution(_camera, _validResolution[0].width, _validResolution[0].height); + ret = camera_set_preview_resolution(_camera, _validPreviewResolution[0].width, _validPreviewResolution[0].height); if (ret != CAMERA_ERROR_NONE) { SINGLEO_LOGE("CameraApiBackend: camera_set_preview_resolution failed. ret: %d", ret); throw InvalidOperation("CameraApiBackend: camera_set_preview_resolution failed."); } + SINGLEO_LOGD("Set camera resulition for capture with width(%d) and height(%d) in default.", + _validCaptureResolution[0].width, _validCaptureResolution[0].height); + + ret = camera_set_capture_resolution(_camera, _validCaptureResolution[0].width, _validCaptureResolution[0].height); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_set_capture_resolution failed. ret: %d", ret); + throw InvalidOperation("CameraApiBackend: camera_set_capture_resolution failed."); + } + ret = camera_foreach_supported_preview_format(_camera, previewFormatCb, this); if (ret != CAMERA_ERROR_NONE) { SINGLEO_LOGE("CameraApiBackend: camera_foreach_supported_preview_format failed. ret: %d", ret); @@ -174,38 +204,130 @@ void CameraApiBackend::configure() } // TODO. read default pixel format from json configuration file later. - auto it = find(_validPixelFormat.begin(), _validPixelFormat.end(), _defaultPixelFormat); - if (it == _validPixelFormat.end()) { - SINGLEO_LOGE("Invalid pixel format type."); - throw InvalidOperation("CameraApiBackend: Invalid pixel format type."); + auto preview_it = + find(_validPreviewPixelFormat.begin(), _validPreviewPixelFormat.end(), _defaultPreviewPixelFormat); + if (preview_it == _validPreviewPixelFormat.end()) { + SINGLEO_LOGE("Invalid preview pixel format type."); + throw InvalidOperation("CameraApiBackend: Invalid preview pixel format type."); } - ret = camera_set_preview_format(_camera, _defaultPixelFormat); + ret = camera_set_preview_format(_camera, _defaultPreviewPixelFormat); if (ret != CAMERA_ERROR_NONE) { SINGLEO_LOGE("CameraApiBackend: camera_set_preview_format failed. ret: %d", ret); throw InvalidOperation("CameraApiBackend: camera_set_preview_format failed."); } - ret = camera_set_preview_cb(_camera, previewCb, this); + ret = camera_foreach_supported_capture_format(_camera, captureFormatCb, this); if (ret != CAMERA_ERROR_NONE) { - SINGLEO_LOGE("CameraApiBackend: camera_set_preview_cb failed. ret: %d", ret); - throw InvalidOperation("CameraApiBackend: camera_set_preview_cb failed."); + SINGLEO_LOGE("CameraApiBackend: camera_foreach_supported_capture_format failed. ret: %d", ret); + throw InvalidOperation("CameraApiBackend: camera_foreach_supported_capture_format failed."); } + + // TODO. read default pixel format from json configuration file later. + auto capture_it = + find(_validCapturePixelFormat.begin(), _validCapturePixelFormat.end(), _defaultCapturePixelFormat); + if (capture_it == _validCapturePixelFormat.end()) { + SINGLEO_LOGE("Invalid capture pixel format type."); + throw InvalidOperation("CameraApiBackend: Invalid capture pixel format type."); + } + + ret = camera_set_capture_format(_camera, _defaultCapturePixelFormat); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_set_capture_format failed. ret: %d", ret); + throw InvalidOperation("CameraApiBackend: camera_set_capture_format failed."); + } +} + +void CameraApiBackend::captureCb(camera_image_data_s *image, camera_image_data_s *postview, + camera_image_data_s *thumbnail, void *user_data) +{ + auto context = static_cast(user_data); + + if (image->format != CAMERA_PIXEL_FORMAT_I420) { + SINGLEO_LOGE("Pixel format not supported."); + throw InvalidParameter("Pixel format not supported."); + } + + // TODO. consider for other pixel format support later. + + cv::Mat cv_src(image->height + image->height / 2, image->width, CV_8UC1, image->data); + + cv::cvtColor(cv_src, context->_cvCaptureImage, cv::COLOR_YUV2BGR_I420); + + context->_capture_event.notify_one(); +} + +void CameraApiBackend::captureCompletedCb(void *user_data) +{ + // Nothing to do } void CameraApiBackend::capture(BaseDataType &out_data) { - throw InvalidOperation("Not supported yet"); + // Capture sequence should "start preview -> start capture -> start preview -> stop preview" + // according to the Camera API state digram below, + // https://docs.tizen.org/application/native/api/mobile/4.0/group__CAPI__MEDIA__CAMERA__MODULE.html + + // Change the state to "Previewing" from "Created". + int ret = camera_start_preview(_camera); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_start_preview failed. ret: %d", ret); + throw InvalidOperation("CameraApiBackend: camera_start_preview failed."); + } + + // Change the state to "Capturing" and then "Captured". + // Ps. captureCb callback will be called by another thread context camera-api internally + // but by main thread for captureCompletedCb callback. + ret = camera_start_capture(_camera, captureCb, captureCompletedCb, this); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_start_capture failed. ret: %d", ret); + throw InvalidOperation("camera_start_preview: camera_start_capture failed."); + } + + unique_lock lock(_capture_mutex); + + _capture_event.wait(lock); + + auto &image_data = dynamic_cast(out_data); + + image_data.width = _cvCaptureImage.cols; + image_data.height = _cvCaptureImage.rows; + image_data.pixel_format = ImagePixelFormat::RGB888; + image_data.byte_per_pixel = _cvCaptureImage.channels(); + image_data.ptr = _cvCaptureImage.data; + + // Wait for "Captured" state. + // Ps. captureCompletedCb callback is called by main thread so we cannot use the callback + // to wait for "Captured" state with sync API. + camera_state_e state {}; + + camera_get_state(_camera, &state); + while (state != CAMERA_STATE_CAPTURED) { + this_thread::sleep_for(10ms); + camera_get_state(_camera, &state); + } + + // Change the state to "Previewing". + ret = camera_start_preview(_camera); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_start_preview failed. ret: %d", ret); + throw InvalidOperation("CameraApiBackend: camera_start_preview failed."); + } + + // Change the state to "Created". + ret = camera_stop_preview(_camera); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_stop_preview failed. ret: %d", ret); + throw InvalidOperation("CameraApiBackend: camera_stop_preview failed."); + } } void CameraApiBackend::threadLoop() { - SINGLEO_LOGD("CameraApiBackend: stream off."); - int ret = camera_start_preview(_camera); if (ret != CAMERA_ERROR_NONE) { SINGLEO_LOGE("CameraApiBackend: camera_start_preview failed. ret: %d", ret); - throw InvalidOperation("camera_start_preview: camera_set_preview_format failed."); + throw InvalidOperation("CameraApiBackend: camera_start_preview failed."); } unique_lock lock(_preview_mutex); @@ -216,6 +338,12 @@ void CameraApiBackend::streamOn() { SINGLEO_LOGD("CameraApiBackend: stream on."); + int ret = camera_set_preview_cb(_camera, previewCb, this); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_set_preview_cb failed. ret: %d", ret); + throw InvalidOperation("CameraApiBackend: camera_set_preview_cb failed."); + } + if (!_thread_handle) _thread_handle = std::make_unique(&CameraApiBackend::threadLoop, this); } @@ -230,6 +358,12 @@ void CameraApiBackend::streamOff() throw InvalidOperation("camera_start_preview: camera_stop_preview failed."); } + ret = camera_unset_preview_cb(_camera); + if (ret != CAMERA_ERROR_NONE) { + SINGLEO_LOGE("CameraApiBackend: camera_unset_preview_cb failed. ret: %d", ret); + throw InvalidOperation("camera_start_preview: camera_unset_preview_cb failed."); + } + _exit_thread = true; _preview_event.notify_one(); _thread_handle->join(); diff --git a/services/auto_zoom/src/AutoZoom.cpp b/services/auto_zoom/src/AutoZoom.cpp index b599d5e..40e21fe 100644 --- a/services/auto_zoom/src/AutoZoom.cpp +++ b/services/auto_zoom/src/AutoZoom.cpp @@ -202,6 +202,7 @@ void AutoZoom::updateResult(BaseDataType &in_data) SINGLEO_LOGD("detected object count = %zu", autozoom_result.num_of_objects); if (autozoom_result.num_of_objects == 0) { + _result = autozoom_result; SINGLEO_LOGW("No detected objects."); return; } @@ -219,7 +220,6 @@ void AutoZoom::getResultCnt(unsigned int *cnt) _postprocessor->addInput(_result); } - // TODO. Temparary code. *cnt = static_cast(_result.num_of_objects); } diff --git a/test/services/test_autozoom.cpp b/test/services/test_autozoom.cpp index 1167e9b..8176d45 100644 --- a/test/services/test_autozoom.cpp +++ b/test/services/test_autozoom.cpp @@ -73,27 +73,31 @@ TEST(AutoZoomTest, InferenceRequestWithCameraInputFeedShouldBeOk) int ret = singleo_service_create("service=auto_zoom, input_feed=camera, camera_id=0, fps=30, async=0", &handle); ASSERT_EQ(ret, SINGLEO_ERROR_NONE); - ret = singleo_service_perform(handle); - ASSERT_EQ(ret, SINGLEO_ERROR_NONE); - - unsigned int cnt; + const unsigned int max_iteration = 10; - ret = singleo_service_get_result_cnt(handle, &cnt); - ASSERT_EQ(ret, SINGLEO_ERROR_NONE); + for (unsigned int repeat_cnt = 0; repeat_cnt < max_iteration; ++repeat_cnt) { + ret = singleo_service_perform(handle); + ASSERT_EQ(ret, SINGLEO_ERROR_NONE); - for (unsigned int idx = 0; idx < cnt; ++idx) { - unsigned int x, y, w, h; + unsigned int cnt; - ret = singleo_service_get_result_int(handle, idx, "x", &x); - ASSERT_EQ(ret, SINGLEO_ERROR_NONE); - ret = singleo_service_get_result_int(handle, idx, "y", &y); - ASSERT_EQ(ret, SINGLEO_ERROR_NONE); - ret = singleo_service_get_result_int(handle, idx, "width", &w); - ASSERT_EQ(ret, SINGLEO_ERROR_NONE); - ret = singleo_service_get_result_int(handle, idx, "height", &h); + ret = singleo_service_get_result_cnt(handle, &cnt); ASSERT_EQ(ret, SINGLEO_ERROR_NONE); - cout << x << " x " << y << " ~ " << w << " x " << h << endl; + for (unsigned int idx = 0; idx < cnt; ++idx) { + unsigned int x, y, w, h; + + ret = singleo_service_get_result_int(handle, idx, "x", &x); + ASSERT_EQ(ret, SINGLEO_ERROR_NONE); + ret = singleo_service_get_result_int(handle, idx, "y", &y); + ASSERT_EQ(ret, SINGLEO_ERROR_NONE); + ret = singleo_service_get_result_int(handle, idx, "width", &w); + ASSERT_EQ(ret, SINGLEO_ERROR_NONE); + ret = singleo_service_get_result_int(handle, idx, "height", &h); + ASSERT_EQ(ret, SINGLEO_ERROR_NONE); + + cout << x << " x " << y << " ~ " << w << " x " << h << endl; + } } ret = singleo_service_destroy(handle);