2 * Copyright (c) 2020 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef GST_USE_UNSTABLE_API
18 #define GST_USE_UNSTABLE_API
19 #include <gst/webrtc/webrtc.h>
21 #include "media_streamer_util.h"
22 #include "media_streamer_priv.h"
23 #include "media_streamer_gst.h"
24 #include "media_streamer_gst_webrtc.h"
25 #include "media_streamer_node.h"
27 static gchar *__make_ice_candidate_message(guint mlineindex, gchar *candidate)
29 JsonObject *ice, *msg;
32 ms_retvm_if(candidate == NULL, NULL, "candidate is NULL");
34 ice = json_object_new();
35 json_object_set_string_member(ice, "candidate", candidate);
36 json_object_set_int_member(ice, "sdpMLineIndex", mlineindex);
38 msg = json_object_new();
39 json_object_set_object_member(msg, "ice", ice);
41 text = ms_get_string_from_json_object(msg);
43 json_object_unref(msg);
48 static gchar* __make_sdp_message(GstWebRTCSessionDescription *desc)
51 JsonObject *msg, *sdp;
53 text = gst_sdp_message_as_text(desc->sdp);
54 sdp = json_object_new();
56 if (desc->type == GST_WEBRTC_SDP_TYPE_OFFER) {
57 ms_info("Making offer message:\n%s", text);
58 json_object_set_string_member(sdp, "type", "offer");
59 } else if (desc->type == GST_WEBRTC_SDP_TYPE_ANSWER) {
60 ms_info("Making answer message:\n%s", text);
61 json_object_set_string_member(sdp, "type", "answer");
63 ms_error("invalid description type");
67 json_object_set_string_member(sdp, "sdp", text);
70 msg = json_object_new();
71 json_object_set_object_member(msg, "sdp", sdp);
73 text = ms_get_string_from_json_object(msg);
75 json_object_unref(msg);
80 static void __trigger_message_callback(media_streamer_node_s *webrtc_node, gchar *message)
82 media_streamer_webrtc_callbacks_s *_callbacks;
84 ms_retm_if(webrtc_node == NULL, "webrtc_node is NULL");
85 ms_retm_if(message == NULL, "message is NULL");
87 ms_debug("message is : \n%s", message);
89 _callbacks = (media_streamer_webrtc_callbacks_s *) webrtc_node->callbacks_structure;
90 if (_callbacks->message_cb.callback) {
91 ms_debug("=====> invoke message callback(%p)", _callbacks->message_cb.callback);
92 ((media_streamer_webrtc_message_cb)(_callbacks->message_cb.callback))(webrtc_node, message, _callbacks->message_cb.user_data);
93 ms_debug("<===== end of the callback");
95 ms_warning("message callback is NULL");
99 static void __ms_webrtcbin_set_session_description(GstElement *webrtcbin, GstWebRTCSessionDescription *session_description, gboolean is_remote)
102 ms_retm_if(session_description == NULL, "session_description is NULL");
103 ms_retm_if(webrtcbin == NULL, "webrtcbin is NULL");
107 promise = gst_promise_new();
108 g_signal_emit_by_name(webrtcbin, is_remote? "set-remote-description" : "set-local-description", session_description, promise);
109 gst_promise_interrupt(promise);
110 gst_promise_unref(promise);
115 static void __on_offer_created_cb(GstPromise *promise, gpointer user_data)
117 GstWebRTCSessionDescription *offer = NULL;
118 const GstStructure *reply;
119 media_streamer_node_s *webrtc_node = (media_streamer_node_s *)user_data;
120 node_info_s *node_klass_type = NULL;
121 GstElement *webrtcbin = NULL;
124 ms_retm_if(promise == NULL, "promise is NULL");
125 ms_retm_if(webrtc_node == NULL, "webrtc_node is NULL");
126 ms_retm_if(gst_promise_wait(promise) != GST_PROMISE_RESULT_REPLIED, "promise is not for replied result");
130 node_klass_type = ms_node_get_klass_by_its_type(MEDIA_STREAMER_NODE_TYPE_WEBRTC);
131 if (!(webrtcbin = ms_find_element_in_bin_by_type(webrtc_node->gst_element, node_klass_type))) {
132 ms_error("Could not find webrtcbin by type[%s, %s]", node_klass_type->klass_name, node_klass_type->default_name);
136 reply = gst_promise_get_reply(promise);
137 gst_structure_get(reply, "offer",
138 GST_TYPE_WEBRTC_SESSION_DESCRIPTION, &offer, NULL);
139 gst_promise_unref(promise);
141 __ms_webrtcbin_set_session_description(webrtcbin, offer, FALSE);
143 sdp_msg = __make_sdp_message(offer);
144 gst_webrtc_session_description_free(offer);
146 /* Send local description to peer */
147 __trigger_message_callback(webrtc_node, sdp_msg);
153 static void __on_answer_created_cb(GstPromise * promise, gpointer user_data)
155 GstWebRTCSessionDescription *answer = NULL;
156 const GstStructure *reply;
157 media_streamer_node_s *node = (media_streamer_node_s *)user_data;
158 node_info_s *node_klass_type = NULL;
160 GstElement *webrtcbin;
162 ms_retm_if(promise == NULL, "promise is NULL");
163 ms_retm_if(node == NULL, "node is NULL");
164 ms_retm_if(node->gst_element == NULL, "webrtc_container is NULL");
165 ms_retm_if(gst_promise_wait(promise) != GST_PROMISE_RESULT_REPLIED, "promise is not for replied result");
169 node_klass_type = ms_node_get_klass_by_its_type(MEDIA_STREAMER_NODE_TYPE_WEBRTC);
170 if (!(webrtcbin = ms_find_element_in_bin_by_type(node->gst_element, node_klass_type))) {
171 ms_error("Could not find webrtcbin by type[%s, %s]", node_klass_type->klass_name, node_klass_type->default_name);
175 reply = gst_promise_get_reply(promise);
176 gst_structure_get(reply, "answer", GST_TYPE_WEBRTC_SESSION_DESCRIPTION, &answer, NULL);
177 gst_promise_unref(promise);
179 __ms_webrtcbin_set_session_description(webrtcbin, answer, FALSE);
181 sdp_msg = __make_sdp_message(answer);
182 gst_webrtc_session_description_free(answer);
184 /* Send local description to peer */
185 __trigger_message_callback(node, sdp_msg);
191 int ms_webrtcbin_set_remote_session_description(media_streamer_node_s *node, const char *sdp_msg)
193 GstSDPMessage *gst_sdp;
196 GstWebRTCSessionDescription *answer, *offer;
197 node_info_s *node_klass_type;
198 GstElement *webrtcbin;
199 int ret = MEDIA_STREAMER_ERROR_NONE;
201 ms_retvm_if(node == NULL || node->gst_element == NULL, MEDIA_STREAMER_ERROR_INVALID_PARAMETER, "Error: empty webrtcbin");
202 ms_retvm_if(sdp_msg == NULL, MEDIA_STREAMER_ERROR_INVALID_PARAMETER, "sdp_msg is NULL");
207 node_klass_type = ms_node_get_klass_by_its_type(MEDIA_STREAMER_NODE_TYPE_WEBRTC);
208 if (!(webrtcbin = ms_find_element_in_bin_by_type(node->gst_element, node_klass_type))) {
209 ms_error("Could not find webrtcbin by type[%s, %s]", node_klass_type->klass_name, node_klass_type->default_name);
210 return MEDIA_STREAMER_ERROR_INVALID_OPERATION;
213 ret = ms_webrtc_get_sdp_from_message(sdp_msg, &sdp, &type);
214 if (ret != MEDIA_STREAMER_ERROR_NONE)
217 ret = gst_sdp_message_new(&gst_sdp);
218 if (ret != GST_SDP_OK) {
219 ret = MEDIA_STREAMER_ERROR_INVALID_OPERATION;
223 ret = gst_sdp_message_parse_buffer((guint8 *)sdp, strlen(sdp), gst_sdp);
224 if (ret != GST_SDP_OK) {
225 ret = MEDIA_STREAMER_ERROR_INVALID_OPERATION;
229 if (g_str_equal(type, "answer")) {
230 answer = gst_webrtc_session_description_new(GST_WEBRTC_SDP_TYPE_ANSWER, gst_sdp);
231 g_assert_nonnull(answer);
233 __ms_webrtcbin_set_session_description(webrtcbin, answer, TRUE);
234 gst_webrtc_session_description_free(answer);
235 } else if (g_str_equal(type, "offer")) {
236 offer = gst_webrtc_session_description_new(GST_WEBRTC_SDP_TYPE_OFFER, gst_sdp);
237 g_assert_nonnull(offer);
239 __ms_webrtcbin_set_session_description(webrtcbin, offer, TRUE);
240 gst_webrtc_session_description_free(offer);
242 ms_webrtcbin_on_negotiation_process_answer(webrtcbin, node);
244 ms_error("type is %s, it is not a answer or offer", type);
256 int ms_webrtcbin_add_ice_candidate(media_streamer_node_s *node, const char *ice_msg)
260 node_info_s *node_klass_type;
261 GstElement *webrtcbin;
264 ms_retvm_if(node == NULL || node->gst_element == NULL, MEDIA_STREAMER_ERROR_INVALID_PARAMETER, "Error: empty webrtcbin");
265 ms_retvm_if(ice_msg == NULL, MEDIA_STREAMER_ERROR_INVALID_PARAMETER, "ice_msg is NULL");
269 node_klass_type = ms_node_get_klass_by_its_type(MEDIA_STREAMER_NODE_TYPE_WEBRTC);
270 if (!(webrtcbin = ms_find_element_in_bin_by_type(node->gst_element, node_klass_type))) {
271 ms_error("Could not find webrtcbin by type[%s, %s]", node_klass_type->klass_name, node_klass_type->default_name);
272 return MEDIA_STREAMER_ERROR_INVALID_OPERATION;
275 ret = ms_webrtc_get_ice_candidate_from_message(ice_msg, &candidate, &sdpmlineindex);
276 if (ret != MEDIA_STREAMER_ERROR_NONE)
279 /*Add ice candidate sent by remote peer */
280 g_signal_emit_by_name(webrtcbin, "add-ice-candidate", sdpmlineindex, candidate);
284 return MEDIA_STREAMER_ERROR_NONE;
287 void ms_webrtcbin_on_negotiation_process_answer(GstElement *webrtcbin, media_streamer_node_s *webrtc_node)
291 ms_retm_if(webrtcbin == NULL, "webrtcbin is NULL");
295 promise = gst_promise_new_with_change_func(__on_answer_created_cb, webrtc_node, NULL);
296 g_signal_emit_by_name(G_OBJECT(webrtcbin), "create-answer", NULL, promise);
301 void ms_webrtcbin_on_negotiation_needed_cb(GstElement *webrtcbin, gpointer user_data)
305 ms_retm_if(webrtcbin == NULL, "webrtcbin is NULL");
306 ms_retm_if(user_data == NULL, "user_data is NULL");
310 promise = gst_promise_new_with_change_func(__on_offer_created_cb, user_data, NULL);
311 g_signal_emit_by_name(G_OBJECT(webrtcbin), "create-offer", NULL, promise);
316 void ms_webrtcbin_on_ice_candidate_cb(GstElement *webrtcbin, guint mlineindex, gchar *candidate, gpointer user_data)
318 gchar *ice_candidate_msg = NULL;
319 media_streamer_node_s *webrtc_node = (media_streamer_node_s *)user_data;
321 ms_retm_if(webrtcbin == NULL, "webrtcbin is NULL");
322 ms_retm_if(candidate == NULL, "candidate is NULL");
323 ms_retm_if(webrtc_node == NULL, "webrtc_node is NULL");
325 ice_candidate_msg = __make_ice_candidate_message(mlineindex, candidate);
327 __trigger_message_callback(webrtc_node, ice_candidate_msg);
329 g_free(ice_candidate_msg);
332 void ms_webrtcbin_notify_ice_gathering_state_cb(GstElement *webrtcbin, GParamSpec * pspec, gpointer user_data)
334 GstWebRTCICEGatheringState ice_gather_state;
335 const gchar *new_state = "UNKNOWN";
337 g_object_get(webrtcbin, "ice-gathering-state", &ice_gather_state, NULL);
339 switch (ice_gather_state) {
340 case GST_WEBRTC_ICE_GATHERING_STATE_NEW:
343 case GST_WEBRTC_ICE_GATHERING_STATE_GATHERING:
344 new_state = "GATHERING";
346 case GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE:
347 new_state = "COMPLETE";
351 ms_info("ICE gathering state changed to [%s]", new_state);
354 static void __data_channel_on_error_cb(GObject *data_channel, gpointer user_data)
356 ms_retm_if(data_channel == NULL, "data_channel is NULL");
363 static void __data_channel_on_open_cb(GObject *data_channel, gpointer user_data)
365 GBytes *bytes = NULL;
367 ms_retm_if(data_channel == NULL, "data_channel is NULL");
371 bytes = g_bytes_new("data", strlen("data"));
373 g_signal_emit_by_name(data_channel, "send-string", "Hi! from GStreamer");
374 g_signal_emit_by_name(data_channel, "send-data", bytes);
376 g_bytes_unref(bytes);
381 static void __data_channel_on_close_cb(GObject *data_channel, gpointer user_data)
383 ms_retm_if(data_channel == NULL, "data_channel is NULL");
390 static void __data_channel_on_message_string_cb(GObject *data_channel, gchar *message, gpointer user_data)
392 ms_retm_if(data_channel == NULL, "data_channel is NULL");
393 ms_retm_if(message == NULL, "message is NULL");
395 ms_info("Received message: %s", message);
398 static void __connect_data_channel_signals(GObject *data_channel)
400 ms_retm_if(data_channel == NULL, "data_channel is NULL");
404 g_signal_connect(data_channel, "on-error", G_CALLBACK(__data_channel_on_error_cb), NULL);
405 g_signal_connect(data_channel, "on-open", G_CALLBACK(__data_channel_on_open_cb), NULL);
406 g_signal_connect(data_channel, "on-close", G_CALLBACK(__data_channel_on_close_cb), NULL);
407 g_signal_connect(data_channel, "on-message-string", G_CALLBACK(__data_channel_on_message_string_cb), NULL);
412 void ms_webrtcbin_on_data_channel_cb(GstElement *webrtcbin, GObject *data_channel, gpointer user_data)
414 media_streamer_s *ms_streamer = (media_streamer_s *)user_data;
416 ms_retm_if(ms_streamer == NULL, "ms_streamer is NULL");
417 ms_retm_if(data_channel == NULL, "data_channel is NULL");
421 __connect_data_channel_signals(data_channel);
426 static void __trigger_decoded_ready_callback(media_streamer_node_s *webrtc_node, const gchar *new_pad_name, const gchar *media_type)
428 media_streamer_webrtc_callbacks_s *_callbacks;
430 ms_retm_if(webrtc_node == NULL, "webrtc_node is NULL");
431 ms_retm_if(new_pad_name == NULL, "new_pad_name is NULL");
432 ms_retm_if(media_type == NULL, "media_type is NULL");
434 _callbacks = (media_streamer_webrtc_callbacks_s *) webrtc_node->callbacks_structure;
435 if (!_callbacks->decoded_ready_cb.callback) {
436 ms_warning("decoded_ready_cb.callback is NULL");
440 ms_debug("=====> invoke decoded ready callback(%p)", _callbacks->decoded_ready_cb.callback);
441 ((media_streamer_node_decoded_ready_cb)(_callbacks->decoded_ready_cb.callback))(webrtc_node,
442 (const char *)new_pad_name,
443 (const char *)media_type,
444 _callbacks->decoded_ready_cb.user_data);
445 ms_debug("<===== end of the callback");
448 static void __decodebin_pad_added_cb(GstElement *decodebin, GstPad *new_pad, gpointer user_data)
450 media_streamer_node_s *webrtc_node = (media_streamer_node_s *)user_data;
451 const gchar *new_pad_name;
452 const gchar *media_type;
454 ms_retm_if(decodebin == NULL, "decodebin is NULL");
455 ms_retm_if(new_pad == NULL, "new_pad is NULL");
456 ms_retm_if(webrtc_node == NULL, "webrtc_node is NULL");
457 ms_retm_if(GST_PAD_DIRECTION(new_pad) != GST_PAD_SRC, "new_pad is not for source");
458 ms_retm_if(gst_pad_has_current_caps(new_pad) == FALSE, "new_pad does not have caps");
460 new_pad_name = GST_PAD_NAME(new_pad);
461 media_type = gst_structure_get_name(gst_caps_get_structure(gst_pad_get_current_caps(new_pad), 0));
463 ms_debug("new_pad_name[%s], media_type[%s]", new_pad_name, media_type);
465 __trigger_decoded_ready_callback(webrtc_node, new_pad_name, media_type);
468 void ms_webrtcbin_pad_added_cb(GstElement *webrtcbin, GstPad *new_pad, gpointer user_data)
470 media_streamer_node_s *webrtc_node = (media_streamer_node_s *)user_data;
471 media_streamer_webrtc_callbacks_s *_callbacks;
473 GstElement *decodebin;
475 ms_retm_if(new_pad == NULL, "new_pad is NULL");
476 ms_retm_if(webrtc_node == NULL, "webrtc_node is NULL");
477 ms_retm_if(webrtc_node->callbacks_structure == NULL, "callbacks_structure is NULL");
478 ms_retm_if(GST_PAD_DIRECTION(new_pad) != GST_PAD_SRC, "new_pad is not for source");
482 ms_debug("Pad [%s] added on [%s]", GST_PAD_NAME(new_pad), GST_ELEMENT_NAME(webrtcbin));
484 _callbacks = (media_streamer_webrtc_callbacks_s *) webrtc_node->callbacks_structure;
485 if (!_callbacks->decoded_ready_cb.callback) {
486 ms_warning("decoded_ready_cb.callback is null, skip it");
490 decodebin = ms_element_create(DEFAULT_DECODEBIN, NULL);
491 ms_retm_if(decodebin == NULL, "decodebin is NULL");
493 ms_retm_if(webrtc_node->parent_streamer == NULL, "parent_streamer is NULL");
494 ms_retm_if(webrtc_node->parent_streamer->pipeline == NULL, "pipeline is NULL");
495 gst_bin_add(GST_BIN(webrtc_node->parent_streamer->pipeline), decodebin);
497 gst_element_sync_state_with_parent(decodebin);
499 g_signal_connect(decodebin, "pad-added", G_CALLBACK(__decodebin_pad_added_cb), webrtc_node);
501 sink_pad = gst_element_get_static_pad(decodebin, "sink");
502 ms_retm_if(sink_pad == NULL, "sink_pad is NULL");
504 gst_pad_link(new_pad, sink_pad);
505 gst_object_unref(sink_pad);
510 GstElement *ms_webrtc_element_create(void)
512 GstElement *webrtc_container;
513 GstElement *webrtcbin;
514 GstGhostPad *ghost_pad_video_in;
518 webrtc_container = gst_bin_new("webrtc_container");
519 ms_retvm_if(!webrtc_container, (GstElement *) NULL, "Error: creating elements for webrtc container");
521 ms_add_no_target_ghostpad(webrtc_container, MS_RTP_PAD_VIDEO_IN, GST_PAD_SINK);
523 MS_SET_INT_STATIC_STRING_PARAM(webrtc_container, MEDIA_STREAMER_PARAM_WEBRTC_PEER_TYPE, DEFAULT_WEBRTC_PEER);
524 MS_SET_INT_STATIC_STRING_PARAM(webrtc_container, MEDIA_STREAMER_PARAM_WEBRTC_REMOTE_SESSION_DESCRIPTION, NULL);
525 MS_SET_INT_STATIC_STRING_PARAM(webrtc_container, MEDIA_STREAMER_PARAM_WEBRTC_ADD_ICE_CANDIDATE, NULL);
527 if (!(webrtcbin = ms_element_create("webrtcbin", NULL))) {
528 ms_error("Failed to create webrtcbin element");
532 /* FIXME: these should be set from user */
533 g_object_set(G_OBJECT(webrtcbin), "bundle-policy", 3, NULL); // 3:max-bundle
534 g_object_set(G_OBJECT(webrtcbin), "stun-server", "stun://stun.l.google.com:19302", NULL);
536 ms_bin_add_element(webrtc_container, webrtcbin, FALSE);
538 if (!(ghost_pad_video_in = (GstGhostPad *)gst_element_get_static_pad(webrtc_container, MS_RTP_PAD_VIDEO_IN))) {
539 ms_error("Failed to get ghost pad for webrtc_container");
543 if (!(gst_ghost_pad_set_target(ghost_pad_video_in, gst_element_get_request_pad(webrtcbin, "sink_%u")))) {
544 ms_info("Failed to gst_ghost_pad_set_target() for %s", MS_RTP_PAD_VIDEO_IN);
545 /* release resources */
551 return webrtc_container;