From f32385dd5a07727a5f5146475a10a5ff3bf48eea Mon Sep 17 00:00:00 2001 From: Kartik Anand Date: Fri, 12 May 2023 17:29:33 +0530 Subject: [PATCH] Implement source types in android WebRTC --- .../main/java/com/samsung/android/aitt/Aitt.java | 21 +--- .../samsung/android/aitt/handler/RTSPHandler.java | 13 +- .../android/aitt/handler/StreamHandler.java | 2 +- .../android/aitt/handler/WebRTCHandler.java | 16 +-- .../samsung/android/aitt/internal/Definitions.java | 1 + .../samsung/android/aitt/stream/AittStream.java | 5 +- .../samsung/android/aitt/stream/RTSPStream.java | 5 +- .../samsung/android/aitt/stream/WebRTCStream.java | 104 ++++++---------- .../modules/webrtc/WebRTCInstrumentedTest.java | 113 +++++++++++++---- .../modules/webrtc/src/main/AndroidManifest.xml | 1 + .../com/samsung/android/modules/webrtc/WebRTC.java | 119 +++++++++++++----- .../android/modules/webrtc/WebRTCPublisher.java | 138 ++++++++++++++++----- .../android/modules/webrtc/WebRTCSubscriber.java | 22 ++-- .../android/modules/webrtc/WebRTCUnitTest.java | 8 +- 14 files changed, 355 insertions(+), 213 deletions(-) diff --git a/android/aitt/src/main/java/com/samsung/android/aitt/Aitt.java b/android/aitt/src/main/java/com/samsung/android/aitt/Aitt.java index f099b76..539f434 100644 --- a/android/aitt/src/main/java/com/samsung/android/aitt/Aitt.java +++ b/android/aitt/src/main/java/com/samsung/android/aitt/Aitt.java @@ -349,16 +349,6 @@ public class Aitt { return hostTable; } - // TODO: Update publish with proper stream interface. - public boolean publish(AittStream stream, String topic, byte[] message) { - if (stream == null) { - Log.e(TAG, "Stream is null."); - return false; - } - - return stream.publish(topic, message); - } - /** * Method to create transportHandler and publish message based on protocol * @@ -776,13 +766,14 @@ public class Aitt { } /** - * Method that receives message from JNI layer for topics other than discovery topics + * Method to create AittStream object * * @param protocol The data received from JNI layer to be sent to application layer * @param topic The data received from JNI layer to be sent to application layer * @param streamRole The data received from JNI layer to be sent to application layer + * @return AittStream instance */ - public AittStream createStream(Protocol protocol, String topic, AittStream.StreamRole streamRole) { + public AittStream createStream(Protocol protocol, String topic, AittStream.StreamRole streamRole) throws InstantiationException { ModuleHandler moduleHandler = createModuleHandler(protocol); if (moduleHandler == null) { Log.e(TAG, "Fail to create a module handler."); @@ -792,13 +783,11 @@ public class Aitt { switch (protocol) { case WEBRTC: WebRTCStream webRTCStream = (WebRTCStream) ((WebRTCHandler) moduleHandler).newStreamModule(protocol, topic, streamRole, appContext); - if (webRTCStream != null) - webRTCStream.setJNIInterface(mJniInterface); + webRTCStream.setJNIInterface(mJniInterface); return webRTCStream; case RTSP: RTSPStream rtspStream = (RTSPStream) ((RTSPHandler) moduleHandler).newStreamModule(protocol, topic, streamRole, appContext); - if (rtspStream != null) - rtspStream.setJNIInterface(mJniInterface); + rtspStream.setJNIInterface(mJniInterface); return rtspStream; default: Log.d(TAG, "Not supported yet."); diff --git a/android/aitt/src/main/java/com/samsung/android/aitt/handler/RTSPHandler.java b/android/aitt/src/main/java/com/samsung/android/aitt/handler/RTSPHandler.java index 2d66a22..4e8f4c0 100644 --- a/android/aitt/src/main/java/com/samsung/android/aitt/handler/RTSPHandler.java +++ b/android/aitt/src/main/java/com/samsung/android/aitt/handler/RTSPHandler.java @@ -42,15 +42,10 @@ public class RTSPHandler extends StreamHandler { if (topic == null || topic.isEmpty()) throw new InvalidParameterException("Invalid topic"); - try { - if (role == AittStream.StreamRole.SUBSCRIBER) { - return createSubscriberStream(topic, role); - } else { - return createPublisherStream(topic, role); - } - } catch (Exception e) { - Log.e(TAG, "Fail to create an AittStream instance."); + if (role == AittStream.StreamRole.SUBSCRIBER) { + return createSubscriberStream(topic, role); + } else { + return createPublisherStream(topic, role); } - return null; } } diff --git a/android/aitt/src/main/java/com/samsung/android/aitt/handler/StreamHandler.java b/android/aitt/src/main/java/com/samsung/android/aitt/handler/StreamHandler.java index bec11dc..485de5a 100644 --- a/android/aitt/src/main/java/com/samsung/android/aitt/handler/StreamHandler.java +++ b/android/aitt/src/main/java/com/samsung/android/aitt/handler/StreamHandler.java @@ -30,7 +30,7 @@ public class StreamHandler implements ModuleHandler { * @param context context of the application * @return returns stream object */ - AittStream newStreamModule(Aitt.Protocol protocol, String topic, AittStream.StreamRole role, Context context) { + AittStream newStreamModule(Aitt.Protocol protocol, String topic, AittStream.StreamRole role, Context context) throws InstantiationException { // TODO: Change this function properly after refactoring WebRTC modules. return null; } diff --git a/android/aitt/src/main/java/com/samsung/android/aitt/handler/WebRTCHandler.java b/android/aitt/src/main/java/com/samsung/android/aitt/handler/WebRTCHandler.java index 7142dc8..5694978 100644 --- a/android/aitt/src/main/java/com/samsung/android/aitt/handler/WebRTCHandler.java +++ b/android/aitt/src/main/java/com/samsung/android/aitt/handler/WebRTCHandler.java @@ -36,20 +36,14 @@ public final class WebRTCHandler extends StreamHandler { } @Override - public AittStream newStreamModule(Aitt.Protocol protocol, String topic, AittStream.StreamRole role, Context context) { + public AittStream newStreamModule(Aitt.Protocol protocol, String topic, AittStream.StreamRole role, Context context) throws InstantiationException { if (protocol != Aitt.Protocol.WEBRTC) throw new InvalidParameterException("Invalid protocol"); - try { - if (role == AittStream.StreamRole.SUBSCRIBER) { - return createSubscriberStream(topic, role, context); - } else { - return createPublisherStream(topic, role, context); - } - } catch (Exception e) { - Log.e(TAG, "Fail to create an AittStream instance."); + if (role == AittStream.StreamRole.SUBSCRIBER) { + return createSubscriberStream(topic, role, context); + } else { + return createPublisherStream(topic, role, context); } - - return null; } } diff --git a/android/aitt/src/main/java/com/samsung/android/aitt/internal/Definitions.java b/android/aitt/src/main/java/com/samsung/android/aitt/internal/Definitions.java index 5f6cfec..2898c45 100644 --- a/android/aitt/src/main/java/com/samsung/android/aitt/internal/Definitions.java +++ b/android/aitt/src/main/java/com/samsung/android/aitt/internal/Definitions.java @@ -27,4 +27,5 @@ public class Definitions { public static final int DEFAULT_IPC_PORT = 0; public static final int DEFAULT_STREAM_WIDTH = 640; public static final int DEFAULT_STREAM_HEIGHT = 480; + public static final int DEFAULT_FPS = 15; } diff --git a/android/aitt/src/main/java/com/samsung/android/aitt/stream/AittStream.java b/android/aitt/src/main/java/com/samsung/android/aitt/stream/AittStream.java index 1bb5754..9ff69fc 100644 --- a/android/aitt/src/main/java/com/samsung/android/aitt/stream/AittStream.java +++ b/android/aitt/src/main/java/com/samsung/android/aitt/stream/AittStream.java @@ -59,12 +59,11 @@ public interface AittStream { void start(); /** - * Method to publish to a topic - * @param topic String topic to which data is published + * Method to push data on a topic * @param message Data to be published * @return returns status */ - boolean publish(String topic, byte[] message); + boolean push(byte[] message); /** * Method to disconnect from the broker diff --git a/android/aitt/src/main/java/com/samsung/android/aitt/stream/RTSPStream.java b/android/aitt/src/main/java/com/samsung/android/aitt/stream/RTSPStream.java index 532f1e8..41b477a 100644 --- a/android/aitt/src/main/java/com/samsung/android/aitt/stream/RTSPStream.java +++ b/android/aitt/src/main/java/com/samsung/android/aitt/stream/RTSPStream.java @@ -189,14 +189,13 @@ public class RTSPStream implements AittStream { } /** - * Method to publish to a topic + * Method to push data on a topic * - * @param topic String topic to which data is published * @param message Data to be published * @return returns status */ @Override - public boolean publish(String topic, byte[] message) { + public boolean push(byte[] message) { // TODO: implement this function. return true; } diff --git a/android/aitt/src/main/java/com/samsung/android/aitt/stream/WebRTCStream.java b/android/aitt/src/main/java/com/samsung/android/aitt/stream/WebRTCStream.java index 7cbd36f..086c0fe 100644 --- a/android/aitt/src/main/java/com/samsung/android/aitt/stream/WebRTCStream.java +++ b/android/aitt/src/main/java/com/samsung/android/aitt/stream/WebRTCStream.java @@ -15,6 +15,7 @@ */ package com.samsung.android.aitt.stream; +import static com.samsung.android.aitt.internal.Definitions.DEFAULT_FPS; import static com.samsung.android.aitt.internal.Definitions.DEFAULT_STREAM_HEIGHT; import static com.samsung.android.aitt.internal.Definitions.DEFAULT_STREAM_WIDTH; @@ -50,51 +51,35 @@ public final class WebRTCStream implements AittStream { private final String watchTopic; private final StreamRole streamRole; private final String id; - private final StreamInfo streamInfo; + private final WebRTC webrtc; - private WebRTC webrtc; private JniInterface jniInterface; private StreamState streamState = StreamState.INIT; private StreamStateChangeCallback stateChangeCallback = null; - private static class StreamInfo { - private int frameWidth = DEFAULT_STREAM_WIDTH; - private int frameHeight = DEFAULT_STREAM_HEIGHT; - } - - WebRTCStream(String topic, StreamRole streamRole, Context context) { + WebRTCStream(String topic, StreamRole streamRole, Context context) throws InstantiationException { this.streamRole = streamRole; id = createId(); if (streamRole == StreamRole.PUBLISHER) { publishTopic = topic + SRC; watchTopic = topic + SINK; - try { - webrtc = new WebRTCPublisher(context); - } catch (InstantiationException e) { - Log.e(TAG, "Failed to create WebRTC instance"); - } + webrtc = new WebRTCPublisher(context, DEFAULT_STREAM_WIDTH, DEFAULT_STREAM_HEIGHT, DEFAULT_FPS); } else { publishTopic = topic + SINK; watchTopic = topic + SRC; - try { - webrtc = new WebRTCSubscriber(context); - } catch (InstantiationException e) { - Log.e(TAG, "Failed to create WebRTC instance"); - } + webrtc = new WebRTCSubscriber(context, DEFAULT_STREAM_WIDTH, DEFAULT_STREAM_HEIGHT, DEFAULT_FPS); } - - streamInfo = new StreamInfo(); } - public static WebRTCStream createSubscriberStream(String topic, StreamRole streamRole, Context context) { + public static WebRTCStream createSubscriberStream(String topic, StreamRole streamRole, Context context) throws InstantiationException { if (streamRole != StreamRole.SUBSCRIBER) throw new IllegalArgumentException("The role of this stream is not subscriber."); return new WebRTCStream(topic, streamRole, context); } - public static WebRTCStream createPublisherStream(String topic, StreamRole streamRole, Context context) { + public static WebRTCStream createPublisherStream(String topic, StreamRole streamRole, Context context) throws InstantiationException { if (streamRole != StreamRole.PUBLISHER) throw new IllegalArgumentException("The role of this stream is not publisher."); @@ -106,27 +91,36 @@ public final class WebRTCStream implements AittStream { if (streamRole == StreamRole.SUBSCRIBER) throw new IllegalArgumentException("The role of this stream is not publisher"); + if (streamState == StreamState.READY) + throw new RuntimeException("Stream is already started, cannot change configuration now"); + if (key == null) throw new IllegalArgumentException("Invalid key"); switch (key) { + case "SOURCE_TYPE": + webrtc.setSourceType(value); + break; case "HEIGHT": int height = Integer.parseInt(value); - if (height == 0) - streamInfo.frameHeight = DEFAULT_STREAM_HEIGHT; - else if (height < 0) + if (height <= 0) throw new IllegalArgumentException("Invalid frame height"); else - streamInfo.frameHeight = height; + webrtc.setFrameHeight(height); break; case "WIDTH": int width = Integer.parseInt(value); - if (width == 0) - streamInfo.frameWidth = DEFAULT_STREAM_WIDTH; - else if (width < 0) + if (width <= 0) throw new IllegalArgumentException("Invalid frame width"); else - streamInfo.frameWidth = width; + webrtc.setFrameWidth(width); + break; + case "FRAME_RATE": + int fps = Integer.parseInt(value); + if (fps <= 0) + throw new IllegalArgumentException("Invalid frame rate"); + else + webrtc.setFrameRate(fps); break; default: throw new IllegalArgumentException("Invalid key"); @@ -136,10 +130,13 @@ public final class WebRTCStream implements AittStream { @Override public void start() { - if (invalidWebRTC()) { + if (streamState == StreamState.READY || streamState == StreamState.PLAYING) { + Log.e(TAG, "Stream already started"); return; } + webrtc.registerIceCandidateAddedCallback(() -> publishDiscoveryMessage(generateDiscoveryMessage())); + webrtc.start(); if (streamRole == StreamRole.PUBLISHER) { FlexBuffersBuilder fbb = new FlexBuffersBuilder(ByteBuffer.allocate(512)); fbb.putString(START); @@ -184,8 +181,6 @@ public final class WebRTCStream implements AittStream { throw new IllegalArgumentException("The given callback is null."); if (streamRole == StreamRole.SUBSCRIBER) { - if (invalidWebRTC()) - return; webrtc.registerDataCallback(streamDataCallback::pushStreamData); } else if (streamRole == StreamRole.PUBLISHER) { Log.e(TAG, "Invalid function call"); @@ -194,42 +189,30 @@ public final class WebRTCStream implements AittStream { @Override public int getStreamHeight() { - if (streamRole == StreamRole.SUBSCRIBER) - return webrtc.getFrameHeight(); - return streamInfo.frameHeight; + return webrtc.getFrameHeight(); } @Override public int getStreamWidth() { - if (streamRole == StreamRole.SUBSCRIBER) - return webrtc.getFrameWidth(); - return streamInfo.frameWidth; + return webrtc.getFrameWidth(); } @Override - public boolean publish(String topic, byte[] message) { - if (invalidWebRTC()) - return false; + public boolean push(byte[] message) { + if (streamRole == StreamRole.SUBSCRIBER) { + throw new RuntimeException("Push is not allowed with a subscriber stream"); + } - if (streamRole == StreamRole.PUBLISHER) { - if (topic.endsWith(Definitions.RESPONSE_POSTFIX)) { - Log.d(TAG, "A message is sent through a WebRTC publisher stream."); - return webrtc.sendMessageData(message); - } else { - Log.d(TAG, "Video data are sent through a WebRTC publisher stream."); - webrtc.sendVideoData(message, streamInfo.frameWidth, streamInfo.frameHeight); - } + if (publishTopic.contains(Definitions.RESPONSE_POSTFIX)) { + Log.d(TAG, "A message is sent through a WebRTC publisher stream."); + return webrtc.pushMessageData(message); } else { - Log.e(TAG, "publish() is not allowed to a subscriber stream."); + Log.d(TAG, "Media-packet is sent through a WebRTC publisher stream."); + return webrtc.pushMediaPacket(message); } - - return true; } public void setJNIInterface(JniInterface jniInterface) { - if (invalidWebRTC()) - return; - this.jniInterface = jniInterface; jniInterface.setDiscoveryCallback(watchTopic, (clientId, status, data) -> { @@ -251,14 +234,6 @@ public final class WebRTCStream implements AittStream { return UUID.randomUUID().toString(); } - private boolean invalidWebRTC() { - if (webrtc == null) { - Log.e(TAG, "Invalid WebRTCStream"); - return true; - } - return false; - } - private void handleStreamState(String clientId, ByteBuffer buffer) { String peerState = FlexBuffers.getRoot(buffer).asString(); if (STOP.compareTo(peerState) == 0) { @@ -340,6 +315,7 @@ public final class WebRTCStream implements AittStream { jniInterface.updateDiscoveryMessage(publishTopic, data); } } + private void updateState(StreamState state) { streamState = state; if (stateChangeCallback != null) diff --git a/android/modules/webrtc/src/androidTest/java/com/samsung/android/modules/webrtc/WebRTCInstrumentedTest.java b/android/modules/webrtc/src/androidTest/java/com/samsung/android/modules/webrtc/WebRTCInstrumentedTest.java index b22c5c8..a7c0e18 100644 --- a/android/modules/webrtc/src/androidTest/java/com/samsung/android/modules/webrtc/WebRTCInstrumentedTest.java +++ b/android/modules/webrtc/src/androidTest/java/com/samsung/android/modules/webrtc/WebRTCInstrumentedTest.java @@ -36,6 +36,7 @@ import android.util.Log; import androidx.annotation.ColorInt; import androidx.test.platform.app.InstrumentationRegistry; +import com.samsung.android.aitt.internal.Definitions; import com.samsung.android.aitt.Aitt; import com.samsung.android.aitt.stream.AittStream; @@ -162,7 +163,7 @@ public class WebRTCInstrumentedTest { Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID, wifiIP, true); publisher.connect(brokerIp, PORT); AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_MESSAGE_TOPIC, PUBLISHER); - publisherStream.start(); + publisherStream.setConfig("SOURCE_TYPE", "MEDIA_PACKET").start(); Log.i(TAG, "A WebRTC client and a publisher stream are created."); subscriberStream.start(); @@ -176,7 +177,7 @@ public class WebRTCInstrumentedTest { while (true) { // TODO: Replace publish - boolean isPublished = publisher.publish(publisherStream, TEST_MESSAGE_TOPIC, message.getBytes()); + boolean isPublished = publisherStream.push(message.getBytes()); if (isPublished) break; } @@ -217,7 +218,7 @@ public class WebRTCInstrumentedTest { Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + LARGE_DATA_PREFIX, wifiIP, true); publisher.connect(brokerIp, PORT); AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_LARGE_MESSAGE_TOPIC, PUBLISHER); - publisherStream.start(); + publisherStream.setConfig("SOURCE_TYPE", "MEDIA_PACKET").start(); Log.i(TAG, "A WebRTC client and a publisher stream are created."); subscriberStream.start(); @@ -231,7 +232,7 @@ public class WebRTCInstrumentedTest { while (true) { // TODO: Replace publish - boolean isPublished = publisher.publish(publisherStream, TEST_LARGE_MESSAGE_TOPIC, largeBytes); + boolean isPublished = publisherStream.push(largeBytes); if (isPublished) break; Thread.sleep(SLEEP_INTERVAL); @@ -250,7 +251,7 @@ public class WebRTCInstrumentedTest { } @Test - public void testWebRTCVideoStreamingNoConfig_N() { + public void testWebRTCMediaPacketStreamImproperConfig_N() { try { Aitt subscriber = new Aitt(appContext, AITT_WEBRTC_SERVER_ID + VIDEO_PREFIX, wifiIP, true); subscriber.connect(brokerIp, PORT); @@ -269,6 +270,7 @@ public class WebRTCInstrumentedTest { Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + VIDEO_PREFIX, wifiIP, true); publisher.connect(brokerIp, PORT); AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, PUBLISHER); + publisherStream.setConfig("SOURCE_TYPE", "MEDIA_PACKET"); publisherStream.start(); Log.i(TAG, "A WebRTC client and a publisher stream are created."); @@ -283,7 +285,7 @@ public class WebRTCInstrumentedTest { while (true) { // TODO: Replace publish - boolean isPublished = publisher.publish(publisherStream, TEST_VIDEO_TOPIC, frameImageBytes); + boolean isPublished = publisherStream.push(frameImageBytes); if (isPublished) break; } @@ -301,7 +303,7 @@ public class WebRTCInstrumentedTest { } @Test - public void testWebRTCVideoStreamingWithConfig_P() { + public void testWebRTCMediaPacketStreamWithConfig_P() { try { Aitt subscriber = new Aitt(appContext, AITT_WEBRTC_SERVER_ID + VIDEO_PREFIX, wifiIP, true); subscriber.connect(brokerIp, PORT); @@ -324,7 +326,8 @@ public class WebRTCInstrumentedTest { publisher.connect(brokerIp, PORT); AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, PUBLISHER); - publisherStream.setConfig("WIDTH", "320") + publisherStream.setConfig("SOURCE_TYPE", "MEDIA_PACKET") + .setConfig("WIDTH", "320") .setConfig("HEIGHT", "240") .start(); @@ -340,7 +343,7 @@ public class WebRTCInstrumentedTest { } while (true) { - boolean isPublished = publisher.publish(publisherStream, TEST_VIDEO_TOPIC, frameImageBytes); + boolean isPublished = publisherStream.push(frameImageBytes); if (isPublished) break; } @@ -358,66 +361,98 @@ public class WebRTCInstrumentedTest { } @Test - public void testWebRTCStreamConfigInvalidKey_N() { + public void testWebRTCCameraStream_P() { try { + Aitt subscriber = new Aitt(appContext, AITT_WEBRTC_SERVER_ID + VIDEO_PREFIX, wifiIP, true); + subscriber.connect(brokerIp, PORT); + AittStream subscriberStream = subscriber.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, SUBSCRIBER); + subscriberStream.setReceiveCallback(data -> { + Log.i(TAG, "A callback is received in testWebRTCVideoStreamingWithConfig"); + if (Definitions.DEFAULT_STREAM_WIDTH != subscriberStream.getStreamWidth()) + throw new RuntimeException("Wrong frame width"); + if (Definitions.DEFAULT_STREAM_HEIGHT != subscriberStream.getStreamHeight()) + throw new RuntimeException("Wrong frame height"); + + Log.i(TAG, "The correct test image is received."); + + if (looper != null) + looper.quit(); + }); + Log.i(TAG, "A WebRTC server and a subscriber stream are created."); + Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + VIDEO_PREFIX, wifiIP, true); publisher.connect(brokerIp, PORT); AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, PUBLISHER); + publisherStream.start(); - assertThrows(IllegalArgumentException.class, () -> publisherStream.setConfig("KEY", "value")); + Log.i(TAG, "A WebRTC client and a publisher stream are created."); + + subscriberStream.start(); + Log.i(TAG, "The subscriber stream starts."); + + int intervalSum = 0; + while (intervalSum < 2000) { + Thread.sleep(SLEEP_INTERVAL); + intervalSum += SLEEP_INTERVAL; + } + + Looper.loop(); + Log.i(TAG, "A looper is finished."); publisherStream.disconnect(); + + subscriberStream.stop(); } catch (Exception e) { - fail("Failed testWebRTCStreamConfigInvalidKey, (" + e + ")"); + fail("Failed testWebRTCVideoStreamingWithConfig, (" + e + ")"); } } @Test - public void testWebRTCStreamConfigNullKey_N() { + public void testWebRTCStreamConfigInvalidKey_N() { try { Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + VIDEO_PREFIX, wifiIP, true); publisher.connect(brokerIp, PORT); AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, PUBLISHER); - assertThrows(IllegalArgumentException.class, () -> publisherStream.setConfig(null, "value")); + assertThrows(IllegalArgumentException.class, () -> publisherStream.setConfig("KEY", "value")); publisherStream.disconnect(); } catch (Exception e) { - fail("Failed testWebRTCStreamConfigNullKey, (" + e + ")"); + fail("Failed testWebRTCStreamConfigInvalidKey, (" + e + ")"); } } @Test - public void testWebRTCStreamConfigWithoutHeight_P() { + public void testWebRTCStreamConfigNullKey_N() { try { Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + VIDEO_PREFIX, wifiIP, true); publisher.connect(brokerIp, PORT); AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, PUBLISHER); - publisherStream.setConfig("WIDTH", "320") - .setConfig("HEIGHT", "0") - .start(); + assertThrows(IllegalArgumentException.class, () -> publisherStream.setConfig(null, "value")); publisherStream.disconnect(); } catch (Exception e) { - fail("Failed testWebRTCStreamConfigWithoutHeight, (" + e + ")"); + fail("Failed testWebRTCStreamConfigNullKey, (" + e + ")"); } } @Test - public void testWebRTCStreamConfigWithoutWidth_P() { + public void testWebRTCStreamSetConfig_P() { try { Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + VIDEO_PREFIX, wifiIP, true); publisher.connect(brokerIp, PORT); AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, PUBLISHER); - publisherStream.setConfig("HEIGHT", "240") - .setConfig("WIDTH", "0") + publisherStream.setConfig("SOURCE_TYPE", "CAMERA") + .setConfig("WIDTH", "320") + .setConfig("HEIGHT", "240") + .setConfig("FRAME_RATE", "30") .start(); publisherStream.disconnect(); } catch (Exception e) { - fail("Failed testWebRTCStreamConfigWithoutWidth, (" + e + ")"); + fail("Failed testWebRTCStreamConfigWithoutHeight, (" + e + ")"); } } @@ -454,6 +489,36 @@ public class WebRTCInstrumentedTest { } @Test + public void testWebRTCStreamInvalidSourceType_N() { + try { + Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + VIDEO_PREFIX, wifiIP, true); + publisher.connect(brokerIp, PORT); + AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, PUBLISHER); + + assertThrows(IllegalArgumentException.class, () -> publisherStream.setConfig("SOURCE_TYPE", "A")); + + publisherStream.disconnect(); + } catch (Exception e) { + fail("Failed testWebRTCStreamInvalidHeight, (" + e + ")"); + } + } + + @Test + public void testWebRTCStreamInvalidFrameRate_N() { + try { + Aitt publisher = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + VIDEO_PREFIX, wifiIP, true); + publisher.connect(brokerIp, PORT); + AittStream publisherStream = publisher.createStream(Aitt.Protocol.WEBRTC, TEST_VIDEO_TOPIC, PUBLISHER); + + assertThrows(IllegalArgumentException.class, () -> publisherStream.setConfig("FRAME_RATE", "-1")); + + publisherStream.disconnect(); + } catch (Exception e) { + fail("Failed testWebRTCStreamInvalidHeight, (" + e + ")"); + } + } + + @Test public void testWebRTCStreamConfigInvalidRole_N() { try { Aitt subscriber = new Aitt(appContext, AITT_WEBRTC_CLIENT_ID + VIDEO_PREFIX, wifiIP, true); diff --git a/android/modules/webrtc/src/main/AndroidManifest.xml b/android/modules/webrtc/src/main/AndroidManifest.xml index ca84321..9ae74c5 100644 --- a/android/modules/webrtc/src/main/AndroidManifest.xml +++ b/android/modules/webrtc/src/main/AndroidManifest.xml @@ -7,6 +7,7 @@ android:required="true" /> + iceCandidates = new ArrayList<>(); protected String peerDiscoveryId; + protected int frameWidth; + protected int frameHeight; + protected int frameRate; + protected SourceType sourceType = SourceType.CAMERA; protected final Context appContext; + private String peerId; /** @@ -71,6 +77,22 @@ public abstract class WebRTC { void onIceCandidate(); } + protected enum SourceType { + CAMERA, + MEDIA_PACKET; + + private static SourceType fromString(String str) { + switch (str) { + case "CAMERA": + return CAMERA; + case "MEDIA_PACKET": + return MEDIA_PACKET; + default: + throw new IllegalArgumentException("Invalid SOURCE_TYPE: " + str); + } + } + } + /** * Method to set remote description of peer * @@ -80,12 +102,17 @@ public abstract class WebRTC { protected abstract void initializePeerConnection(); + protected abstract void configureStream(); + /** * WebRTC constructor to create webRTC instance * * @param appContext Application context creating webRTC instance + * @param width Frame width + * @param height Frame height + * @param fps Frames per second */ - public WebRTC(Context appContext) throws InstantiationException { + public WebRTC(Context appContext, int width, int height, int fps) throws InstantiationException { if (appContext == null) throw new IllegalArgumentException("App context is null."); @@ -94,6 +121,10 @@ public abstract class WebRTC { initializePeerConnection(); if (peerConnection == null) throw new InstantiationException("Failed to create peer connection"); + + frameWidth = width; + frameHeight = height; + frameRate = fps; } /** @@ -120,6 +151,10 @@ public abstract class WebRTC { iceCandidateAddedCallback = cb; } + public void start() { + Log.d(TAG, "Start WebRTC"); + } + /** * Method to disconnect the connection from peer */ @@ -224,24 +259,63 @@ public abstract class WebRTC { stop(); } + public void setSourceType(String type) { + SourceType newType = SourceType.fromString(type); + if (sourceType == newType) + return; + sourceType = newType; + configureStream(); + } + + public void setFrameHeight(int height) { + frameHeight = height; + } + + public void setFrameWidth(int width) { + frameWidth = width; + } + + /** + * Method to get received Frame height + * + * @return Received frame height + */ + public int getFrameHeight() { + return frameHeight; + } + + /** + * Method to get received Frame width + * + * @return Received frame width + */ + public int getFrameWidth() { + return frameWidth; + } + + public void setFrameRate(int fps) { + frameRate = fps; + } + /** - * Method used to send video data + * Method used to push media data * - * @param frame Video frame in byte format - * @param width width of the video frame - * @param height height of the video frame + * @param packet Media packet in byte format + * @return true if message is successfully sent, false otherwise */ - public void sendVideoData(byte[] frame, int width, int height) { - Log.d(TAG, "Send video data"); + public boolean pushMediaPacket(byte[] packet) { + Log.d(TAG, "Push video data"); + return true; } /** - * Method to send message data, if the message size is greater than MAX_MESSAGE_SIZE, message will be compressed before sending + * Method to push message data, if the message size is greater than MAX_MESSAGE_SIZE, message will be compressed before sending * * @param message message to be sent in byte format + * @return true if message is successfully sent, false otherwise */ - public boolean sendMessageData(byte[] message) { - Log.d(TAG, "Send message data"); + public boolean pushMessageData(byte[] message) { + Log.d(TAG, "Push message data"); return true; } @@ -271,10 +345,9 @@ public abstract class WebRTC { * Method to initialize peer connection factory */ private void initializePeerConnectionFactory() { - EglBase mRootEglBase; - mRootEglBase = EglBase.create(); - VideoEncoderFactory encoderFactory = new DefaultVideoEncoderFactory(mRootEglBase.getEglBaseContext(), true , true); - VideoDecoderFactory decoderFactory = new DefaultVideoDecoderFactory(mRootEglBase.getEglBaseContext()); + eglBase = EglBase.create(); + VideoEncoderFactory encoderFactory = new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true , true); + VideoDecoderFactory decoderFactory = new DefaultVideoDecoderFactory(eglBase.getEglBaseContext()); PeerConnectionFactory.initialize(PeerConnectionFactory.InitializationOptions.builder(appContext).setEnableInternalTracer(true).createInitializationOptions()); PeerConnectionFactory.Builder builder = PeerConnectionFactory.builder().setVideoEncoderFactory(encoderFactory).setVideoDecoderFactory(decoderFactory); @@ -308,22 +381,4 @@ public abstract class WebRTC { Log.d(TAG, "onSetFailure: Reason = " + s); } } - - /** - * Method to get received Frame height - * - * @return Received frame height - */ - public int getFrameHeight() { - return 0; - } - - /** - * Method to get received Frame width - * - * @return Received frame width - */ - public int getFrameWidth() { - return 0; - } } diff --git a/android/modules/webrtc/src/main/java/com/samsung/android/modules/webrtc/WebRTCPublisher.java b/android/modules/webrtc/src/main/java/com/samsung/android/modules/webrtc/WebRTCPublisher.java index b07be00..0b7a6d9 100644 --- a/android/modules/webrtc/src/main/java/com/samsung/android/modules/webrtc/WebRTCPublisher.java +++ b/android/modules/webrtc/src/main/java/com/samsung/android/modules/webrtc/WebRTCPublisher.java @@ -27,6 +27,9 @@ import com.github.luben.zstd.Zstd; import org.json.JSONException; import org.json.JSONObject; +import org.webrtc.Camera1Enumerator; +import org.webrtc.Camera2Enumerator; +import org.webrtc.CameraEnumerator; import org.webrtc.CapturerObserver; import org.webrtc.DataChannel; import org.webrtc.IceCandidate; @@ -48,6 +51,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; public final class WebRTCPublisher extends WebRTC { @@ -55,17 +59,19 @@ public final class WebRTCPublisher extends WebRTC { private static final String TAG = "WebRTCPublisher"; private static final String VIDEO_TRACK_ID = "AITT_WEBRTC_v0"; private static final String MEDIA_STREAM_ID = "AITT_WEBRTC"; - - private VideoTrack videoTrackFromSource; - private FrameVideoCapturer videoCapturer; - - public WebRTCPublisher(Context appContext) throws InstantiationException { - super(appContext); - - VideoSource videoSource = connectionFactory.createVideoSource(false); - createVideoTrack(videoSource); - addVideoTrack(); - createVideoCapturer(videoSource); + private static final String SURFACE_TEXTURE_THREAD = "AITT_WEBRTC_SURFACE_TEXTURE"; + + private MediaStream mediaStream; + private VideoSource videoSource; + private VideoTrack videoTrack; + private MediaPacketCapturer mediaPacketCapturer; + private VideoCapturer cameraCapturer; + private CameraEnumerator cameraEnumerator; + private List cameraDeviceNames; + + public WebRTCPublisher(Context appContext, int width, int height, int fps) throws InstantiationException { + super(appContext, width, height, fps); + configureStream(); } @Override @@ -79,12 +85,20 @@ public final class WebRTCPublisher extends WebRTC { } @Override - public void sendVideoData(byte[] frame, int width, int height) { - videoCapturer.send(frame, width, height); + public void start() { + if (sourceType == SourceType.CAMERA && cameraCapturer != null) + cameraCapturer.startCapture(frameWidth, frameHeight, frameRate); + } + + @Override + public boolean pushMediaPacket(byte[] packet) { + if (sourceType != SourceType.MEDIA_PACKET) + throw new RuntimeException("Push is not supported for this source type"); + return mediaPacketCapturer.send(packet, frameWidth, frameHeight); } @Override - public boolean sendMessageData(byte[] message) { + public boolean pushMessageData(byte[] message) { ByteBuffer chunkData; if (message.length < MAX_MESSAGE_SIZE) { ByteBuffer data = ByteBuffer.wrap(message); @@ -202,6 +216,40 @@ public final class WebRTCPublisher extends WebRTC { localDataChannel = peerConnection.createDataChannel("sendDataChannel", new DataChannel.Init()); } + @Override + protected void configureStream() { + if (peerConnection == null) + throw new RuntimeException("Invalid stream state, peer connection not created"); + + cleanStream(); + videoSource = connectionFactory.createVideoSource(false); + createVideoTrack(); + createMediaStream(); + createVideoCapturer(); + } + + private void cleanStream() { + if (mediaStream != null) { + peerConnection.removeStream(mediaStream); + if (videoTrack != null) { + mediaStream.removeTrack(videoTrack); + videoTrack.dispose(); + } + } + + if (videoSource != null) + videoSource.dispose(); + + if (cameraCapturer != null) { + try { + cameraCapturer.stopCapture(); + cameraCapturer.dispose(); + } catch (InterruptedException e) { + Log.e(TAG, "Failed to stop cameraCapturer:" + e.getMessage()); + } + } + } + private void createAnswer() { peerConnection.createAnswer(new SimpleSdpObserver() { @Override @@ -223,40 +271,64 @@ public final class WebRTCPublisher extends WebRTC { }, new MediaConstraints()); } - private void createVideoCapturer(VideoSource videoSource) { - videoCapturer = new FrameVideoCapturer(); - videoCapturer.initialize(null, null, videoSource.getCapturerObserver()); + private void createVideoTrack() { + videoTrack = connectionFactory.createVideoTrack(VIDEO_TRACK_ID, videoSource); + videoTrack.setEnabled(true); } - /** - * Method to create video track - */ - private void createVideoTrack(VideoSource videoSource) { - videoTrackFromSource = connectionFactory.createVideoTrack(VIDEO_TRACK_ID, videoSource); - videoTrackFromSource.setEnabled(true); + private void createMediaStream() { + mediaStream = connectionFactory.createLocalMediaStream(MEDIA_STREAM_ID); + mediaStream.addTrack(videoTrack); + peerConnection.addStream(mediaStream); } - /** - * Method to add video track - */ - private void addVideoTrack() { - MediaStream mediaStream = connectionFactory.createLocalMediaStream(MEDIA_STREAM_ID); - mediaStream.addTrack(videoTrackFromSource); - peerConnection.addStream(mediaStream); + private void createVideoCapturer() { + if (sourceType == SourceType.CAMERA) { + if (cameraDeviceNames == null) + setCameraDeviceNames(); + for (String deviceName : cameraDeviceNames) { + cameraCapturer = cameraEnumerator.createCapturer(deviceName , null); + if (cameraCapturer != null) + break; + } + if (cameraCapturer != null) { + SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create(SURFACE_TEXTURE_THREAD, eglBase.getEglBaseContext()); + cameraCapturer.initialize(surfaceTextureHelper, appContext, videoSource.getCapturerObserver()); + } + } else { + mediaPacketCapturer = new MediaPacketCapturer(); + mediaPacketCapturer.initialize(null, null, videoSource.getCapturerObserver()); + } + } + + private void setCameraDeviceNames() { + cameraDeviceNames = new ArrayList<>(); + if (Camera2Enumerator.isSupported(appContext)) + cameraEnumerator = new Camera2Enumerator(appContext); + else + cameraEnumerator = new Camera1Enumerator(true); + + String[] deviceNames = cameraEnumerator.getDeviceNames(); + for (String deviceName : deviceNames) { + if(cameraEnumerator.isFrontFacing(deviceName)){ + cameraDeviceNames.add(deviceName); + } + } } /** - * Class to implement Frame video capturer + * Class to implement Media Packet capturer */ - private static class FrameVideoCapturer implements VideoCapturer { + private static class MediaPacketCapturer implements VideoCapturer { private CapturerObserver capturerObserver; - void send(byte[] frame, int width, int height) { + boolean send(byte[] frame, int width, int height) { long timestampNS = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); NV21Buffer buffer = new NV21Buffer(frame, width, height, null); VideoFrame videoFrame = new VideoFrame(buffer, 0, timestampNS); this.capturerObserver.onFrameCaptured(videoFrame); videoFrame.release(); + return true; } @Override diff --git a/android/modules/webrtc/src/main/java/com/samsung/android/modules/webrtc/WebRTCSubscriber.java b/android/modules/webrtc/src/main/java/com/samsung/android/modules/webrtc/WebRTCSubscriber.java index 64c2f00..530c635 100644 --- a/android/modules/webrtc/src/main/java/com/samsung/android/modules/webrtc/WebRTCSubscriber.java +++ b/android/modules/webrtc/src/main/java/com/samsung/android/modules/webrtc/WebRTCSubscriber.java @@ -40,7 +40,6 @@ import org.webrtc.VideoTrack; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -50,11 +49,9 @@ public final class WebRTCSubscriber extends WebRTC { private final ByteArrayOutputStream baos; private boolean recvLargeChunk = false; - private int frameHeight = 0; - private int frameWidth = 0; - public WebRTCSubscriber(Context appContext) throws InstantiationException { - super(appContext); + public WebRTCSubscriber(Context appContext, int width, int height, int fps) throws InstantiationException { + super(appContext, width, height, fps); baos = new ByteArrayOutputStream(); } @@ -80,16 +77,6 @@ public final class WebRTCSubscriber extends WebRTC { } @Override - public int getFrameHeight() { - return frameHeight; - } - - @Override - public int getFrameWidth() { - return frameWidth; - } - - @Override protected void initializePeerConnection() { PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(new ArrayList<>()); @@ -208,6 +195,11 @@ public final class WebRTCSubscriber extends WebRTC { localDataChannel = peerConnection.createDataChannel("DataChannel", new DataChannel.Init()); } + @Override + protected void configureStream() { + Log.e(TAG, "Configuring stream not supported for subscriber"); + } + private void createOffer() { MediaConstraints sdpMediaConstraints = new MediaConstraints(); sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); diff --git a/android/modules/webrtc/src/test/java/com/samsung/android/modules/webrtc/WebRTCUnitTest.java b/android/modules/webrtc/src/test/java/com/samsung/android/modules/webrtc/WebRTCUnitTest.java index 35c5e7c..7d22ecc 100644 --- a/android/modules/webrtc/src/test/java/com/samsung/android/modules/webrtc/WebRTCUnitTest.java +++ b/android/modules/webrtc/src/test/java/com/samsung/android/modules/webrtc/WebRTCUnitTest.java @@ -17,6 +17,10 @@ public class WebRTCUnitTest { @Mock private final Context context = mock(Context.class); + private static final int WIDTH = 320; + private static final int HEIGHT = 240; + private static final int FPS = 30; + private static MockedStatic mockedLog; @BeforeClass @@ -32,7 +36,7 @@ public class WebRTCUnitTest { @Test(expected = IllegalArgumentException.class) public void testWebRTCPublisherConstructor_N() throws IllegalArgumentException { try { - WebRTC webRTC = new WebRTCPublisher(null); + WebRTC webRTC = new WebRTCPublisher(null, WIDTH, HEIGHT, FPS); assertNotNull("WebRTC instance not null", webRTC); } catch (InstantiationException e) { fail("Failed to create WebRTCPublisher" + e); @@ -42,7 +46,7 @@ public class WebRTCUnitTest { @Test(expected = IllegalArgumentException.class) public void testWebRTCSubscriberConstructor_N() throws IllegalArgumentException { try { - WebRTC webRTC = new WebRTCSubscriber(null); + WebRTC webRTC = new WebRTCSubscriber(null, WIDTH, HEIGHT, FPS); assertNotNull("WebRTC instance not null", webRTC); } catch (InstantiationException e) { fail("Failed to create WebRTCSubscriber" + e); -- 2.7.4