2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
11 #ifndef WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_
12 #define WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_
14 #include "webrtc/common_video/interface/i420_video_frame.h"
15 #include "webrtc/modules/interface/module.h"
16 #include "webrtc/modules/interface/module_common_types.h"
17 #include "webrtc/modules/video_coding/main/interface/video_coding_defines.h"
18 #include "webrtc/system_wrappers/interface/event_wrapper.h"
24 class EncodedImageCallback;
27 struct CodecSpecificInfo;
31 virtual ~EventFactory() {}
33 virtual EventWrapper* CreateEvent() = 0;
36 class EventFactoryImpl : public EventFactory {
38 virtual ~EventFactoryImpl() {}
40 virtual EventWrapper* CreateEvent() {
41 return EventWrapper::Create();
45 // Used to indicate which decode with errors mode should be used.
46 enum VCMDecodeErrorMode {
47 kNoErrors, // Never decode with errors. Video will freeze
48 // if nack is disabled.
49 kSelectiveErrors, // Frames that are determined decodable in
50 // VCMSessionInfo may be decoded with missing
51 // packets. As not all incomplete frames will be
52 // decodable, video will freeze if nack is disabled.
53 kWithErrors // Release frames as needed. Errors may be
54 // introduced as some encoded frames may not be
58 class VideoCodingModule : public Module
67 enum ReceiverRobustness {
75 static VideoCodingModule* Create();
77 static VideoCodingModule* Create(Clock* clock, EventFactory* event_factory);
79 static void Destroy(VideoCodingModule* module);
81 // Get number of supported codecs
83 // Return value : Number of supported codecs
84 static uint8_t NumberOfCodecs();
86 // Get supported codec settings with using id
89 // - listId : Id or index of the codec to look up
90 // - codec : Memory where the codec settings will be stored
92 // Return value : VCM_OK, on success
93 // VCM_PARAMETER_ERROR if codec not supported or id too high
94 static int32_t Codec(const uint8_t listId, VideoCodec* codec);
96 // Get supported codec settings using codec type
99 // - codecType : The codec type to get settings for
100 // - codec : Memory where the codec settings will be stored
102 // Return value : VCM_OK, on success
103 // VCM_PARAMETER_ERROR if codec not supported
104 static int32_t Codec(VideoCodecType codecType, VideoCodec* codec);
110 // Any encoder-related state of VCM will be initialized to the
111 // same state as when the VCM was created. This will not interrupt
112 // or effect decoding functionality of VCM. VCM will lose all the
113 // encoding-related settings by calling this function.
114 // For instance, a send codec has to be registered again.
116 // Return value : VCM_OK, on success.
118 virtual int32_t InitializeSender() = 0;
120 // Registers a codec to be used for encoding. Calling this
121 // API multiple times overwrites any previously registered codecs.
124 // - sendCodec : Settings for the codec to be registered.
125 // - numberOfCores : The number of cores the codec is allowed
127 // - maxPayloadSize : The maximum size each payload is allowed
128 // to have. Usually MTU - overhead.
130 // Return value : VCM_OK, on success.
132 virtual int32_t RegisterSendCodec(const VideoCodec* sendCodec,
133 uint32_t numberOfCores,
134 uint32_t maxPayloadSize) = 0;
136 // API to get the current send codec in use.
139 // - currentSendCodec : Address where the sendCodec will be written.
141 // Return value : VCM_OK, on success.
143 virtual int32_t SendCodec(VideoCodec* currentSendCodec) const = 0;
145 // API to get the current send codec type
147 // Return value : Codec type, on success.
148 // kVideoCodecUnknown, on error or if no send codec is set
149 virtual VideoCodecType SendCodec() const = 0;
151 // Register an external encoder object. This can not be used together with
152 // external decoder callbacks.
155 // - externalEncoder : Encoder object to be used for encoding frames inserted
156 // with the AddVideoFrame API.
157 // - payloadType : The payload type bound which this encoder is bound to.
159 // Return value : VCM_OK, on success.
161 virtual int32_t RegisterExternalEncoder(VideoEncoder* externalEncoder,
163 bool internalSource = false) = 0;
165 // API to get codec config parameters to be sent out-of-band to a receiver.
168 // - buffer : Memory where the codec config parameters should be written.
169 // - size : Size of the memory available.
171 // Return value : Number of bytes written, on success.
173 virtual int32_t CodecConfigParameters(uint8_t* buffer, int32_t size) = 0;
175 // API to get currently configured encoder target bitrate in bits/s.
177 // Return value : 0, on success.
179 virtual int Bitrate(unsigned int* bitrate) const = 0;
181 // API to get currently configured encoder target frame rate.
183 // Return value : 0, on success.
185 virtual int FrameRate(unsigned int* framerate) const = 0;
187 // Sets the parameters describing the send channel. These parameters are inputs to the
188 // Media Optimization inside the VCM and also specifies the target bit rate for the
189 // encoder. Bit rate used by NACK should already be compensated for by the user.
192 // - target_bitrate : The target bitrate for VCM in bits/s.
193 // - lossRate : Fractions of lost packets the past second.
194 // (loss rate in percent = 100 * packetLoss / 255)
195 // - rtt : Current round-trip time in ms.
197 // Return value : VCM_OK, on success.
199 virtual int32_t SetChannelParameters(uint32_t target_bitrate,
203 // Sets the parameters describing the receive channel. These parameters are inputs to the
204 // Media Optimization inside the VCM.
207 // - rtt : Current round-trip time in ms.
208 // with the most amount available bandwidth in a conference
211 // Return value : VCM_OK, on success.
213 virtual int32_t SetReceiveChannelParameters(uint32_t rtt) = 0;
215 // Register a transport callback which will be called to deliver the encoded data and
219 // - transport : The callback object to register.
221 // Return value : VCM_OK, on success.
223 virtual int32_t RegisterTransportCallback(VCMPacketizationCallback* transport) = 0;
225 // Register video output information callback which will be called to deliver information
226 // about the video stream produced by the encoder, for instance the average frame rate and
230 // - outputInformation : The callback object to register.
232 // Return value : VCM_OK, on success.
234 virtual int32_t RegisterSendStatisticsCallback(
235 VCMSendStatisticsCallback* sendStats) = 0;
237 // Register a video quality settings callback which will be called when
238 // frame rate/dimensions need to be updated for video quality optimization
241 // - videoQMSettings : The callback object to register.
243 // Return value : VCM_OK, on success.
245 virtual int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* videoQMSettings) = 0;
247 // Register a video protection callback which will be called to deliver
248 // the requested FEC rate and NACK status (on/off).
251 // - protection : The callback object to register.
253 // Return value : VCM_OK, on success.
255 virtual int32_t RegisterProtectionCallback(VCMProtectionCallback* protection) = 0;
257 // Enable or disable a video protection method.
260 // - videoProtection : The method to enable or disable.
261 // - enable : True if the method should be enabled, false if
262 // it should be disabled.
264 // Return value : VCM_OK, on success.
266 virtual int32_t SetVideoProtection(VCMVideoProtection videoProtection,
269 // Add one raw video frame to the encoder. This function does all the necessary
270 // processing, then decides what frame type to encode, or if the frame should be
271 // dropped. If the frame should be encoded it passes the frame to the encoder
272 // before it returns.
275 // - videoFrame : Video frame to encode.
276 // - codecSpecificInfo : Extra codec information, e.g., pre-parsed in-band signaling.
278 // Return value : VCM_OK, on success.
280 virtual int32_t AddVideoFrame(
281 const I420VideoFrame& videoFrame,
282 const VideoContentMetrics* contentMetrics = NULL,
283 const CodecSpecificInfo* codecSpecificInfo = NULL) = 0;
285 // Next frame encoded should be an intra frame (keyframe).
287 // Return value : VCM_OK, on success.
289 virtual int32_t IntraFrameRequest(int stream_index) = 0;
291 // Frame Dropper enable. Can be used to disable the frame dropping when the encoder
292 // over-uses its bit rate. This API is designed to be used when the encoded frames
293 // are supposed to be stored to an AVI file, or when the I420 codec is used and the
294 // target bit rate shouldn't affect the frame rate.
297 // - enable : True to enable the setting, false to disable it.
299 // Return value : VCM_OK, on success.
301 virtual int32_t EnableFrameDropper(bool enable) = 0;
303 // Sent frame counters
304 virtual int32_t SentFrameCount(VCMFrameCount& frameCount) const = 0;
310 // The receiver state of the VCM will be initialized to the
311 // same state as when the VCM was created. This will not interrupt
312 // or effect the send side functionality of VCM. VCM will lose all the
313 // decoding-related settings by calling this function. All frames
314 // inside the jitter buffer are flushed and the delay is reset.
315 // For instance, a receive codec has to be registered again.
317 // Return value : VCM_OK, on success.
319 virtual int32_t InitializeReceiver() = 0;
321 // Register possible receive codecs, can be called multiple times for different codecs.
322 // The module will automatically switch between registered codecs depending on the
323 // payload type of incoming frames. The actual decoder will be created when needed.
326 // - receiveCodec : Settings for the codec to be registered.
327 // - numberOfCores : Number of CPU cores that the decoder is allowed to use.
328 // - requireKeyFrame : Set this to true if you don't want any delta frames
329 // to be decoded until the first key frame has been decoded.
331 // Return value : VCM_OK, on success.
333 virtual int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec,
334 int32_t numberOfCores,
335 bool requireKeyFrame = false) = 0;
337 // Register an externally defined decoder/renderer object. Can be a decoder only or a
338 // decoder coupled with a renderer. Note that RegisterReceiveCodec must be called to
339 // be used for decoding incoming streams.
342 // - externalDecoder : The external decoder/renderer object.
343 // - payloadType : The payload type which this decoder should be
345 // - internalRenderTiming : True if the internal renderer (if any) of the decoder
346 // object can make sure to render at a given time in ms.
348 // Return value : VCM_OK, on success.
350 virtual int32_t RegisterExternalDecoder(VideoDecoder* externalDecoder,
352 bool internalRenderTiming) = 0;
354 // Register a receive callback. Will be called whenever there is a new frame ready
358 // - receiveCallback : The callback object to be used by the module when a
359 // frame is ready for rendering.
360 // De-register with a NULL pointer.
362 // Return value : VCM_OK, on success.
364 virtual int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback) = 0;
366 // Register a receive statistics callback which will be called to deliver information
367 // about the video stream received by the receiving side of the VCM, for instance the
368 // average frame rate and bit rate.
371 // - receiveStats : The callback object to register.
373 // Return value : VCM_OK, on success.
375 virtual int32_t RegisterReceiveStatisticsCallback(
376 VCMReceiveStatisticsCallback* receiveStats) = 0;
378 // Register a decoder timing callback which will be called to deliver
379 // information about the timing of the decoder in the receiving side of the
380 // VCM, for instance the current and maximum frame decode latency.
383 // - decoderTiming : The callback object to register.
385 // Return value : VCM_OK, on success.
387 virtual int32_t RegisterDecoderTimingCallback(
388 VCMDecoderTimingCallback* decoderTiming) = 0;
390 // Register a frame type request callback. This callback will be called when the
391 // module needs to request specific frame types from the send side.
394 // - frameTypeCallback : The callback object to be used by the module when
395 // requesting a specific type of frame from the send side.
396 // De-register with a NULL pointer.
398 // Return value : VCM_OK, on success.
400 virtual int32_t RegisterFrameTypeCallback(
401 VCMFrameTypeCallback* frameTypeCallback) = 0;
403 // Registers a callback which is called whenever the receive side of the VCM
404 // encounters holes in the packet sequence and needs packets to be retransmitted.
407 // - callback : The callback to be registered in the VCM.
409 // Return value : VCM_OK, on success.
411 virtual int32_t RegisterPacketRequestCallback(
412 VCMPacketRequestCallback* callback) = 0;
414 // Waits for the next frame in the jitter buffer to become complete
415 // (waits no longer than maxWaitTimeMs), then passes it to the decoder for decoding.
416 // Should be called as often as possible to get the most out of the decoder.
418 // Return value : VCM_OK, on success.
420 virtual int32_t Decode(uint16_t maxWaitTimeMs = 200) = 0;
422 // Registers a callback which conveys the size of the render buffer.
423 virtual int RegisterRenderBufferSizeCallback(
424 VCMRenderBufferSizeCallback* callback) = 0;
426 // Waits for the next frame in the dual jitter buffer to become complete
427 // (waits no longer than maxWaitTimeMs), then passes it to the dual decoder
428 // for decoding. This will never trigger a render callback. Should be
429 // called frequently, and as long as it returns 1 it should be called again
430 // as soon as possible.
432 // Return value : 1, if a frame was decoded
433 // 0, if no frame was decoded
435 virtual int32_t DecodeDualFrame(uint16_t maxWaitTimeMs = 200) = 0;
437 // Reset the decoder state to the initial state.
439 // Return value : VCM_OK, on success.
441 virtual int32_t ResetDecoder() = 0;
443 // API to get the codec which is currently used for decoding by the module.
446 // - currentReceiveCodec : Settings for the codec to be registered.
448 // Return value : VCM_OK, on success.
450 virtual int32_t ReceiveCodec(VideoCodec* currentReceiveCodec) const = 0;
452 // API to get the codec type currently used for decoding by the module.
454 // Return value : codecy type, on success.
455 // kVideoCodecUnknown, on error or if no receive codec is registered
456 virtual VideoCodecType ReceiveCodec() const = 0;
458 // Insert a parsed packet into the receiver side of the module. Will be placed in the
459 // jitter buffer waiting for the frame to become complete. Returns as soon as the packet
460 // has been placed in the jitter buffer.
463 // - incomingPayload : Payload of the packet.
464 // - payloadLength : Length of the payload.
465 // - rtpInfo : The parsed header.
467 // Return value : VCM_OK, on success.
469 virtual int32_t IncomingPacket(const uint8_t* incomingPayload,
470 uint32_t payloadLength,
471 const WebRtcRTPHeader& rtpInfo) = 0;
473 // Minimum playout delay (Used for lip-sync). This is the minimum delay required
474 // to sync with audio. Not included in VideoCodingModule::Delay()
478 // - minPlayoutDelayMs : Additional delay in ms.
480 // Return value : VCM_OK, on success.
482 virtual int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) = 0;
484 // Set the time required by the renderer to render a frame.
487 // - timeMS : The time in ms required by the renderer to render a frame.
489 // Return value : VCM_OK, on success.
491 virtual int32_t SetRenderDelay(uint32_t timeMS) = 0;
493 // The total delay desired by the VCM. Can be less than the minimum
494 // delay set with SetMinimumPlayoutDelay.
496 // Return value : Total delay in ms, on success.
498 virtual int32_t Delay() const = 0;
500 // Get the received frame counters. Keeps track of the number of each frame type
501 // received since the start of the call.
504 // - frameCount : Struct to be filled with the number of frames received.
506 // Return value : VCM_OK, on success.
508 virtual int32_t ReceivedFrameCount(VCMFrameCount& frameCount) const = 0;
510 // Returns the number of packets discarded by the jitter buffer due to being
511 // too late. This can include duplicated packets which arrived after the
512 // frame was sent to the decoder. Therefore packets which were prematurely
513 // NACKed will be counted.
514 virtual uint32_t DiscardedPackets() const = 0;
519 // Set the sender RTX/NACK mode.
521 // - mode : the selected NACK mode.
523 // Return value : VCM_OK, on success;
525 virtual int SetSenderNackMode(SenderNackMode mode) = 0;
527 // Set the sender reference picture selection (RPS) mode.
529 // - enable : true or false, for enable and disable, respectively.
531 // Return value : VCM_OK, on success;
533 virtual int SetSenderReferenceSelection(bool enable) = 0;
535 // Set the sender forward error correction (FEC) mode.
537 // - enable : true or false, for enable and disable, respectively.
539 // Return value : VCM_OK, on success;
541 virtual int SetSenderFEC(bool enable) = 0;
543 // Set the key frame period, or disable periodic key frames (I-frames).
545 // - periodMs : period in ms; <= 0 to disable periodic key frames.
547 // Return value : VCM_OK, on success;
549 virtual int SetSenderKeyFramePeriod(int periodMs) = 0;
551 // Set the receiver robustness mode. The mode decides how the receiver
552 // responds to losses in the stream. The type of counter-measure (soft or
553 // hard NACK, dual decoder, RPS, etc.) is selected through the
554 // robustnessMode parameter. The errorMode parameter decides if it is
555 // allowed to display frames corrupted by losses. Note that not all
556 // combinations of the two parameters are feasible. An error will be
557 // returned for invalid combinations.
559 // - robustnessMode : selected robustness mode.
560 // - errorMode : selected error mode.
562 // Return value : VCM_OK, on success;
564 virtual int SetReceiverRobustnessMode(ReceiverRobustness robustnessMode,
565 VCMDecodeErrorMode errorMode) = 0;
567 // Set the decode error mode. The mode decides which errors (if any) are
568 // allowed in decodable frames. Note that setting decode_error_mode to
569 // anything other than kWithErrors without enabling nack will cause
570 // long-term freezes (resulting from frequent key frame requests) if
571 // packet loss occurs.
572 virtual void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) = 0;
574 // Sets the maximum number of sequence numbers that we are allowed to NACK
575 // and the oldest sequence number that we will consider to NACK. If a
576 // sequence number older than |max_packet_age_to_nack| is missing
577 // a key frame will be requested. A key frame will also be requested if the
578 // time of incomplete or non-continuous frames in the jitter buffer is above
579 // |max_incomplete_time_ms|.
580 virtual void SetNackSettings(size_t max_nack_list_size,
581 int max_packet_age_to_nack,
582 int max_incomplete_time_ms) = 0;
584 // Setting a desired delay to the VCM receiver. Video rendering will be
585 // delayed by at least desired_delay_ms.
586 virtual int SetMinReceiverDelay(int desired_delay_ms) = 0;
588 // Enables recording of debugging information.
589 virtual int StartDebugRecording(const char* file_name_utf8) = 0;
591 // Disables recording of debugging information.
592 virtual int StopDebugRecording() = 0;
594 // Lets the sender suspend video when the rate drops below
595 // |threshold_bps|, and turns back on when the rate goes back up above
596 // |threshold_bps| + |window_bps|.
597 virtual void SuspendBelowMinBitrate() = 0;
599 // Returns true if SuspendBelowMinBitrate is engaged and the video has been
600 // suspended due to bandwidth limitations; otherwise false.
601 virtual bool VideoSuspended() const = 0;
603 virtual void RegisterPreDecodeImageCallback(
604 EncodedImageCallback* observer) = 0;
605 virtual void RegisterPostEncodeImageCallback(
606 EncodedImageCallback* post_encode_callback) = 0;
609 } // namespace webrtc
611 #endif // WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_