From 0ebcf43f2235017a6844948ef6d26c74865083aa Mon Sep 17 00:00:00 2001 From: Denis Kormalev Date: Wed, 8 Jan 2014 17:06:59 +0400 Subject: [PATCH] Android: QVideoProbe support for camera QMediaVideoProbeControl sublclass added to capture service to make QVideoProbe work with Android camera. [ChangeLog][QtMultimedia][Android] QVideoProbe support for camera Task-number: QTBUG-35416 Change-Id: I14d0a0e8abd14ee8f577e2901b976b8ed46eb320 Reviewed-by: Yoann Lopes --- .../qtproject/qt5/android/multimedia/QtCamera.java | 11 ++++ .../android/src/mediacapture/mediacapture.pri | 6 ++- .../src/mediacapture/qandroidcamerasession.cpp | 52 +++++++++++++++++-- .../src/mediacapture/qandroidcamerasession.h | 11 ++++ .../src/mediacapture/qandroidcaptureservice.cpp | 30 +++++++++-- .../qandroidmediavideoprobecontrol.cpp | 59 ++++++++++++++++++++++ .../mediacapture/qandroidmediavideoprobecontrol.h | 59 ++++++++++++++++++++++ src/plugins/android/src/wrappers/jcamera.cpp | 29 ++++++++++- src/plugins/android/src/wrappers/jcamera.h | 2 + 9 files changed, 248 insertions(+), 11 deletions(-) create mode 100644 src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.cpp create mode 100644 src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.h diff --git a/src/plugins/android/jar/src/org/qtproject/qt5/android/multimedia/QtCamera.java b/src/plugins/android/jar/src/org/qtproject/qt5/android/multimedia/QtCamera.java index 4aa07b7..f03053f 100644 --- a/src/plugins/android/jar/src/org/qtproject/qt5/android/multimedia/QtCamera.java +++ b/src/plugins/android/jar/src/org/qtproject/qt5/android/multimedia/QtCamera.java @@ -60,6 +60,7 @@ public class QtCamera implements Camera.ShutterCallback, private int m_actualPreviewBuffer = 0; private final ReentrantLock m_buffersLock = new ReentrantLock(); private boolean m_isReleased = false; + private boolean m_fetchEachFrame = false; private static final String TAG = "Qt Camera"; @@ -141,6 +142,11 @@ public class QtCamera implements Camera.ShutterCallback, } } + public void fetchEachFrame(boolean fetch) + { + m_fetchEachFrame = fetch; + } + public void startPreview() { Camera.Size previewSize = m_camera.getParameters().getPreviewSize(); @@ -233,6 +239,10 @@ public class QtCamera implements Camera.ShutterCallback, public void onPreviewFrame(byte[] data, Camera camera) { m_buffersLock.lock(); + + if (data != null && m_fetchEachFrame) + notifyFrameFetched(m_cameraId, data); + if (data == m_cameraPreviewFirstBuffer) m_actualPreviewBuffer = 1; else if (data == m_cameraPreviewSecondBuffer) @@ -252,4 +262,5 @@ public class QtCamera implements Camera.ShutterCallback, private static native void notifyAutoFocusComplete(int id, boolean success); private static native void notifyPictureExposed(int id); private static native void notifyPictureCaptured(int id, byte[] data); + private static native void notifyFrameFetched(int id, byte[] data); } diff --git a/src/plugins/android/src/mediacapture/mediacapture.pri b/src/plugins/android/src/mediacapture/mediacapture.pri index d994beb..242afe6 100644 --- a/src/plugins/android/src/mediacapture/mediacapture.pri +++ b/src/plugins/android/src/mediacapture/mediacapture.pri @@ -21,7 +21,8 @@ SOURCES += \ $$PWD/qandroidaudioencodersettingscontrol.cpp \ $$PWD/qandroidmediacontainercontrol.cpp \ $$PWD/qandroidvideoencodersettingscontrol.cpp \ - $$PWD/qandroidaudioinputselectorcontrol.cpp + $$PWD/qandroidaudioinputselectorcontrol.cpp \ + $$PWD/qandroidmediavideoprobecontrol.cpp HEADERS += \ $$PWD/qandroidcaptureservice.h \ @@ -44,4 +45,5 @@ HEADERS += \ $$PWD/qandroidaudioencodersettingscontrol.h \ $$PWD/qandroidmediacontainercontrol.h \ $$PWD/qandroidvideoencodersettingscontrol.h \ - $$PWD/qandroidaudioinputselectorcontrol.h + $$PWD/qandroidaudioinputselectorcontrol.h \ + $$PWD/qandroidmediavideoprobecontrol.h diff --git a/src/plugins/android/src/mediacapture/qandroidcamerasession.cpp b/src/plugins/android/src/mediacapture/qandroidcamerasession.cpp index cb52823..86c5e31 100644 --- a/src/plugins/android/src/mediacapture/qandroidcamerasession.cpp +++ b/src/plugins/android/src/mediacapture/qandroidcamerasession.cpp @@ -44,11 +44,13 @@ #include "jcamera.h" #include "jmultimediautils.h" #include "qandroidvideooutput.h" +#include "qandroidmediavideoprobecontrol.h" #include "qandroidmultimediautils.h" #include #include #include #include +#include QT_BEGIN_NAMESPACE @@ -183,6 +185,9 @@ bool QAndroidCameraSession::open() if (m_camera) { connect(m_camera, SIGNAL(pictureExposed()), this, SLOT(onCameraPictureExposed())); connect(m_camera, SIGNAL(previewFetched(QByteArray)), this, SLOT(onCameraPreviewFetched(QByteArray))); + connect(m_camera, SIGNAL(frameFetched(QByteArray)), + this, SLOT(onCameraFrameFetched(QByteArray)), + Qt::DirectConnection); connect(m_camera, SIGNAL(pictureCaptured(QByteArray)), this, SLOT(onCameraPictureCaptured(QByteArray))); connect(m_camera, SIGNAL(previewStarted()), this, SLOT(onCameraPreviewStarted())); connect(m_camera, SIGNAL(previewStopped()), this, SLOT(onCameraPreviewStopped())); @@ -200,6 +205,8 @@ bool QAndroidCameraSession::open() if (m_camera->getPreviewFormat() != JCamera::NV21) m_camera->setPreviewFormat(JCamera::NV21); + m_camera->fetchEachFrame(m_videoProbes.count()); + emit opened(); } else { m_status = QCamera::UnavailableStatus; @@ -364,6 +371,25 @@ int QAndroidCameraSession::currentCameraRotation() const return rotation; } +void QAndroidCameraSession::addProbe(QAndroidMediaVideoProbeControl *probe) +{ + m_videoProbesMutex.lock(); + if (probe) + m_videoProbes << probe; + if (m_camera) + m_camera->fetchEachFrame(m_videoProbes.count()); + m_videoProbesMutex.unlock(); +} + +void QAndroidCameraSession::removeProbe(QAndroidMediaVideoProbeControl *probe) +{ + m_videoProbesMutex.lock(); + m_videoProbes.remove(probe); + if (m_camera) + m_camera->fetchEachFrame(m_videoProbes.count()); + m_videoProbesMutex.unlock(); +} + void QAndroidCameraSession::applyImageSettings() { if (!m_camera || !m_imageSettingsDirty) @@ -513,6 +539,19 @@ void QAndroidCameraSession::onCameraPreviewFetched(const QByteArray &preview) } } +void QAndroidCameraSession::onCameraFrameFetched(const QByteArray &frame) +{ + m_videoProbesMutex.lock(); + if (frame.size() && m_videoProbes.count()) { + QVideoFrame videoFrame(new DataVideoBuffer(frame), + m_camera->previewSize(), + QVideoFrame::Format_NV21); + foreach (QAndroidMediaVideoProbeControl *probe, m_videoProbes) + probe->newFrameProbed(videoFrame); + } + m_videoProbesMutex.unlock(); +} + void QAndroidCameraSession::onCameraPictureCaptured(const QByteArray &data) { if (!m_captureCanceled) { @@ -593,10 +632,15 @@ void QAndroidCameraSession::processCapturedImage(int id, void QAndroidCameraSession::processPreviewImage(int id, const QByteArray &data, int rotation) { + emit imageCaptured(id, prepareImageFromPreviewData(data, rotation)); +} + +QImage QAndroidCameraSession::prepareImageFromPreviewData(const QByteArray &data, int rotation) +{ QSize frameSize = m_camera->previewSize(); - QImage preview(frameSize, QImage::Format_ARGB32); + QImage result(frameSize, QImage::Format_ARGB32); qt_convert_NV21_to_ARGB32((const uchar *)data.constData(), - (quint32 *)preview.bits(), + (quint32 *)result.bits(), frameSize.width(), frameSize.height()); @@ -610,9 +654,9 @@ void QAndroidCameraSession::processPreviewImage(int id, const QByteArray &data, transform.rotate(rotation); - preview = preview.transformed(transform); + result = result.transformed(transform); - emit imageCaptured(id, preview); + return result; } void QAndroidCameraSession::onVideoOutputReady(bool ready) diff --git a/src/plugins/android/src/mediacapture/qandroidcamerasession.h b/src/plugins/android/src/mediacapture/qandroidcamerasession.h index a621404..e0fd93c 100644 --- a/src/plugins/android/src/mediacapture/qandroidcamerasession.h +++ b/src/plugins/android/src/mediacapture/qandroidcamerasession.h @@ -45,12 +45,15 @@ #include #include #include +#include +#include #include "qandroidmediastoragelocation.h" QT_BEGIN_NAMESPACE class JCamera; class QAndroidVideoOutput; +class QAndroidMediaVideoProbeControl; class QAndroidCameraSession : public QObject { @@ -90,6 +93,9 @@ public: int currentCameraRotation() const; + void addProbe(QAndroidMediaVideoProbeControl *probe); + void removeProbe(QAndroidMediaVideoProbeControl *probe); + Q_SIGNALS: void statusChanged(QCamera::Status status); void stateChanged(QCamera::State); @@ -114,6 +120,7 @@ private Q_SLOTS: void onCameraPictureExposed(); void onCameraPreviewFetched(const QByteArray &preview); + void onCameraFrameFetched(const QByteArray &frame); void onCameraPictureCaptured(const QByteArray &data); void onCameraPreviewStarted(); void onCameraPreviewStopped(); @@ -127,6 +134,7 @@ private: void applyImageSettings(); void processPreviewImage(int id, const QByteArray &data, int rotation); + QImage prepareImageFromPreviewData(const QByteArray &data, int rotation); void processCapturedImage(int id, const QByteArray &data, const QSize &resolution, @@ -156,6 +164,9 @@ private: QString m_currentImageCaptureFileName; QAndroidMediaStorageLocation m_mediaStorageLocation; + + QSet m_videoProbes; + QMutex m_videoProbesMutex; }; QT_END_NAMESPACE diff --git a/src/plugins/android/src/mediacapture/qandroidcaptureservice.cpp b/src/plugins/android/src/mediacapture/qandroidcaptureservice.cpp index 3468c64..4a38ff5 100644 --- a/src/plugins/android/src/mediacapture/qandroidcaptureservice.cpp +++ b/src/plugins/android/src/mediacapture/qandroidcaptureservice.cpp @@ -61,6 +61,7 @@ #include "qandroidaudioencodersettingscontrol.h" #include "qandroidvideoencodersettingscontrol.h" #include "qandroidmediacontainercontrol.h" +#include "qandroidmediavideoprobecontrol.h" #include @@ -201,16 +202,37 @@ QMediaControl *QAndroidCaptureService::requestControl(const char *name) return m_videoRendererControl; } + if (qstrcmp(name,QMediaVideoProbeControl_iid) == 0) { + QAndroidMediaVideoProbeControl *videoProbe = 0; + if (m_cameraSession) { + videoProbe = new QAndroidMediaVideoProbeControl(this); + m_cameraSession->addProbe(videoProbe); + } + return videoProbe; + } + return 0; } void QAndroidCaptureService::releaseControl(QMediaControl *control) { - if (control && control == m_videoRendererControl) { - m_cameraSession->setVideoPreview(0); - delete m_videoRendererControl; - m_videoRendererControl = 0; + if (control) { + if (control == m_videoRendererControl) { + m_cameraSession->setVideoPreview(0); + delete m_videoRendererControl; + m_videoRendererControl = 0; + return; + } + + QAndroidMediaVideoProbeControl *videoProbe = qobject_cast(control); + if (videoProbe) { + if (m_cameraSession) + m_cameraSession->removeProbe(videoProbe); + delete videoProbe; + return; + } } + } QT_END_NAMESPACE diff --git a/src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.cpp b/src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.cpp new file mode 100644 index 0000000..a4ebc8f --- /dev/null +++ b/src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.cpp @@ -0,0 +1,59 @@ +/**************************************************************************** +** +** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +** Copyright (C) 2013 Integrated Computer Solutions, Inc +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qandroidmediavideoprobecontrol.h" +#include + +QAndroidMediaVideoProbeControl::QAndroidMediaVideoProbeControl(QObject *parent) : + QMediaVideoProbeControl(parent) +{ +} + +QAndroidMediaVideoProbeControl::~QAndroidMediaVideoProbeControl() +{ + +} + +void QAndroidMediaVideoProbeControl::newFrameProbed(const QVideoFrame &frame) +{ + emit videoFrameProbed(frame); +} diff --git a/src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.h b/src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.h new file mode 100644 index 0000000..7bb34d8 --- /dev/null +++ b/src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.h @@ -0,0 +1,59 @@ +/**************************************************************************** +** +** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +** Copyright (C) 2013 Integrated Computer Solutions, Inc +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QANDROIDMEDIAVIDEOPROBECONTROL_H +#define QANDROIDMEDIAVIDEOPROBECONTROL_H + +#include + +class QAndroidMediaVideoProbeControl : public QMediaVideoProbeControl +{ + Q_OBJECT +public: + explicit QAndroidMediaVideoProbeControl(QObject *parent = 0); + virtual ~QAndroidMediaVideoProbeControl(); + + void newFrameProbed(const QVideoFrame& frame); + +}; + +#endif // QANDROIDMEDIAVIDEOPROBECONTROL_H diff --git a/src/plugins/android/src/wrappers/jcamera.cpp b/src/plugins/android/src/wrappers/jcamera.cpp index 0c7455f..d24a59a 100644 --- a/src/plugins/android/src/wrappers/jcamera.cpp +++ b/src/plugins/android/src/wrappers/jcamera.cpp @@ -111,6 +111,21 @@ static void notifyPictureCaptured(JNIEnv *env, jobject, int id, jbyteArray data) } } +static void notifyFrameFetched(JNIEnv *env, jobject, int id, jbyteArray data) +{ + g_objectMapMutex.lock(); + JCamera *obj = g_objectMap.value(id, 0); + g_objectMapMutex.unlock(); + if (obj) { + QByteArray bytes; + int arrayLength = env->GetArrayLength(data); + bytes.resize(arrayLength); + env->GetByteArrayRegion(data, 0, arrayLength, (jbyte*)bytes.data()); + + Q_EMIT obj->frameFetched(bytes); + } +} + class JCameraInstantiator : public QObject { Q_OBJECT @@ -201,6 +216,7 @@ class JCameraWorker : public QObject, public QJNIObjectPrivate Q_INVOKABLE void startPreview(); Q_INVOKABLE void stopPreview(); + Q_INVOKABLE void fetchEachFrame(bool fetch); Q_INVOKABLE void fetchLastPreviewFrame(); Q_INVOKABLE void applyParameters(); @@ -571,6 +587,11 @@ void JCamera::takePicture() QMetaObject::invokeMethod(d, "callVoidMethod", Q_ARG(QByteArray, "takePicture")); } +void JCamera::fetchEachFrame(bool fetch) +{ + QMetaObject::invokeMethod(d, "fetchEachFrame", Q_ARG(bool, fetch)); +} + void JCamera::fetchLastPreviewFrame() { QMetaObject::invokeMethod(d, "fetchLastPreviewFrame"); @@ -1165,6 +1186,11 @@ void JCameraWorker::stopPreview() emit previewStopped(); } +void JCameraWorker::fetchEachFrame(bool fetch) +{ + callMethod("fetchEachFrame", "(Z)V", fetch); +} + void JCameraWorker::fetchLastPreviewFrame() { QJNIEnvironmentPrivate env; @@ -1224,7 +1250,8 @@ void JCameraWorker::callVoidMethod(const QByteArray &methodName) static JNINativeMethod methods[] = { {"notifyAutoFocusComplete", "(IZ)V", (void *)notifyAutoFocusComplete}, {"notifyPictureExposed", "(I)V", (void *)notifyPictureExposed}, - {"notifyPictureCaptured", "(I[B)V", (void *)notifyPictureCaptured} + {"notifyPictureCaptured", "(I[B)V", (void *)notifyPictureCaptured}, + {"notifyFrameFetched", "(I[B)V", (void *)notifyFrameFetched} }; bool JCamera::initJNI(JNIEnv *env) diff --git a/src/plugins/android/src/wrappers/jcamera.h b/src/plugins/android/src/wrappers/jcamera.h index 9b2f5b1..535efe2 100644 --- a/src/plugins/android/src/wrappers/jcamera.h +++ b/src/plugins/android/src/wrappers/jcamera.h @@ -155,6 +155,7 @@ public: void takePicture(); + void fetchEachFrame(bool fetch); void fetchLastPreviewFrame(); QJNIObjectPrivate getCameraObject(); @@ -173,6 +174,7 @@ Q_SIGNALS: void pictureExposed(); void pictureCaptured(const QByteArray &data); void previewFetched(const QByteArray &preview); + void frameFetched(const QByteArray &frame); private: JCamera(int cameraId, jobject cam, QThread *workerThread); -- 2.7.4