2 // Copyright 2010 Google Inc.
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
7 // 1. Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // 2. Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // 3. The name of the author may not be used to endorse or promote products
13 // derived from this software without specific prior written permission.
15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 #ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT
27 #define TALK_MEDIA_BASE_VIDEOADAPTER_H_
29 #include "talk/base/common.h" // For ASSERT
30 #include "talk/base/criticalsection.h"
31 #include "talk/base/logging.h"
32 #include "talk/base/scoped_ptr.h"
33 #include "talk/base/sigslot.h"
34 #include "talk/media/base/videocommon.h"
40 // VideoAdapter adapts an input video frame to an output frame based on the
41 // specified input and output formats. The adaptation includes dropping frames
42 // to reduce frame rate and scaling frames. VideoAdapter is thread safe.
46 virtual ~VideoAdapter();
48 virtual void SetInputFormat(const VideoFormat& format);
49 void SetOutputFormat(const VideoFormat& format);
50 // Constrain output resolution to this many pixels overall
51 void SetOutputNumPixels(int num_pixels);
52 int GetOutputNumPixels() const;
54 const VideoFormat& input_format();
55 const VideoFormat& output_format();
56 // If the parameter black is true, the adapted frames will be black.
57 void SetBlackOutput(bool black);
59 // Adapt the input frame from the input format to the output format. Return
60 // true and set the output frame to NULL if the input frame is dropped. Return
61 // true and set the out frame to output_frame_ if the input frame is adapted
62 // successfully. Return false otherwise.
63 // output_frame_ is owned by the VideoAdapter that has the best knowledge on
65 bool AdaptFrame(const VideoFrame* in_frame, VideoFrame** out_frame);
67 void set_scale_third(bool enable) {
68 LOG(LS_INFO) << "Video Adapter third scaling is now "
69 << (enable ? "enabled" : "disabled");
70 scale_third_ = enable;
72 bool scale_third() const { return scale_third_; }
75 float FindClosestScale(int width, int height, int target_num_pixels);
76 float FindClosestViewScale(int width, int height, int target_num_pixels);
77 float FindLowerScale(int width, int height, int target_num_pixels);
80 const float* GetViewScaleFactors() const;
81 float FindScale(const float* scale_factors,
82 const float upbias, int width, int height,
83 int target_num_pixels);
84 bool StretchToOutputFrame(const VideoFrame* in_frame);
86 VideoFormat input_format_;
87 VideoFormat output_format_;
88 int output_num_pixels_;
89 bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3.
90 int frames_in_; // Number of input frames.
91 int frames_out_; // Number of output frames.
92 int frames_scaled_; // Number of frames scaled.
93 int adaption_changes_; // Number of changes in scale factor.
94 size_t previous_width_; // Previous adapter output width.
95 size_t previous_height_; // Previous adapter output height.
96 bool black_output_; // Flag to tell if we need to black output_frame_.
97 bool is_black_; // Flag to tell if output_frame_ is currently black.
98 int64 interval_next_frame_;
99 talk_base::scoped_ptr<VideoFrame> output_frame_;
100 // The critical section to protect the above variables.
101 talk_base::CriticalSection critical_section_;
103 DISALLOW_COPY_AND_ASSIGN(VideoAdapter);
106 // CoordinatedVideoAdapter adapts the video input to the encoder by coordinating
107 // the format request from the server, the resolution request from the encoder,
109 class CoordinatedVideoAdapter
110 : public VideoAdapter, public sigslot::has_slots<> {
112 enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE };
113 enum AdaptReasonEnum {
115 ADAPTREASON_BANDWIDTH = 2,
118 typedef int AdaptReason;
120 CoordinatedVideoAdapter();
121 virtual ~CoordinatedVideoAdapter() {}
123 virtual void SetInputFormat(const VideoFormat& format);
125 // Enable or disable video adaptation due to the change of the CPU load.
126 void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; }
127 bool cpu_adaptation() const { return cpu_adaptation_; }
128 // Enable or disable smoothing when doing CPU adaptation. When smoothing is
129 // enabled, system CPU load is tracked using an exponential weighted
131 void set_cpu_smoothing(bool enable) {
132 LOG(LS_INFO) << "CPU smoothing is now "
133 << (enable ? "enabled" : "disabled");
134 cpu_smoothing_ = enable;
136 bool cpu_smoothing() const { return cpu_smoothing_; }
137 // Enable or disable video adaptation due to the change of the GD
138 void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; }
139 bool gd_adaptation() const { return gd_adaptation_; }
140 // Enable or disable video adaptation due to the change of the View
141 void set_view_adaptation(bool enable) { view_adaptation_ = enable; }
142 bool view_adaptation() const { return view_adaptation_; }
143 // Enable or disable video adaptation to fast switch View
144 void set_view_switch(bool enable) { view_switch_ = enable; }
145 bool view_switch() const { return view_switch_; }
147 CoordinatedVideoAdapter::AdaptReason adapt_reason() const {
148 return adapt_reason_;
151 // When the video is decreased, set the waiting time for CPU adaptation to
152 // decrease video again.
153 void set_cpu_load_min_samples(int cpu_load_min_samples) {
154 if (cpu_load_min_samples_ != cpu_load_min_samples) {
155 LOG(LS_INFO) << "VAdapt Change Cpu Adapt Min Samples from: "
156 << cpu_load_min_samples_ << " to "
157 << cpu_load_min_samples;
158 cpu_load_min_samples_ = cpu_load_min_samples;
161 int cpu_load_min_samples() const { return cpu_load_min_samples_; }
162 // CPU system load high threshold for reducing resolution. e.g. 0.85f
163 void set_high_system_threshold(float high_system_threshold) {
164 ASSERT(high_system_threshold <= 1.0f);
165 ASSERT(high_system_threshold >= 0.0f);
166 if (high_system_threshold_ != high_system_threshold) {
167 LOG(LS_INFO) << "VAdapt Change High System Threshold from: "
168 << high_system_threshold_ << " to " << high_system_threshold;
169 high_system_threshold_ = high_system_threshold;
172 float high_system_threshold() const { return high_system_threshold_; }
173 // CPU system load low threshold for increasing resolution. e.g. 0.70f
174 void set_low_system_threshold(float low_system_threshold) {
175 ASSERT(low_system_threshold <= 1.0f);
176 ASSERT(low_system_threshold >= 0.0f);
177 if (low_system_threshold_ != low_system_threshold) {
178 LOG(LS_INFO) << "VAdapt Change Low System Threshold from: "
179 << low_system_threshold_ << " to " << low_system_threshold;
180 low_system_threshold_ = low_system_threshold;
183 float low_system_threshold() const { return low_system_threshold_; }
184 // CPU process load threshold for reducing resolution. e.g. 0.10f
185 void set_process_threshold(float process_threshold) {
186 ASSERT(process_threshold <= 1.0f);
187 ASSERT(process_threshold >= 0.0f);
188 if (process_threshold_ != process_threshold) {
189 LOG(LS_INFO) << "VAdapt Change High Process Threshold from: "
190 << process_threshold_ << " to " << process_threshold;
191 process_threshold_ = process_threshold;
194 float process_threshold() const { return process_threshold_; }
196 // Handle the format request from the server via Jingle update message.
197 void OnOutputFormatRequest(const VideoFormat& format);
198 // Handle the resolution request from the encoder due to bandwidth changes.
199 void OnEncoderResolutionRequest(int width, int height, AdaptRequest request);
200 // Handle the resolution request for CPU overuse.
201 void OnCpuResolutionRequest(AdaptRequest request);
202 // Handle the CPU load provided by a CPU monitor.
203 void OnCpuLoadUpdated(int current_cpus, int max_cpus,
204 float process_load, float system_load);
206 sigslot::signal0<> SignalCpuAdaptationUnable;
209 // Adapt to the minimum of the formats the server requests, the CPU wants, and
210 // the encoder wants. Returns true if resolution changed.
211 bool AdaptToMinimumFormat(int* new_width, int* new_height);
212 bool IsMinimumFormat(int pixels);
213 void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,
215 CoordinatedVideoAdapter::AdaptRequest FindCpuRequest(
216 int current_cpus, int max_cpus,
217 float process_load, float system_load);
219 bool cpu_adaptation_; // True if cpu adaptation is enabled.
220 bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation).
221 bool gd_adaptation_; // True if gd adaptation is enabled.
222 bool view_adaptation_; // True if view adaptation is enabled.
223 bool view_switch_; // True if view switch is enabled.
224 int cpu_downgrade_count_;
225 int cpu_load_min_samples_;
226 int cpu_load_num_samples_;
227 // cpu system load thresholds relative to max cpus.
228 float high_system_threshold_;
229 float low_system_threshold_;
230 // cpu process load thresholds relative to current cpus.
231 float process_threshold_;
232 // Video formats that the server view requests, the CPU wants, and the encoder
233 // wants respectively. The adapted output format is the minimum of these.
234 int view_desired_num_pixels_;
235 int64 view_desired_interval_;
236 int encoder_desired_num_pixels_;
237 int cpu_desired_num_pixels_;
238 CoordinatedVideoAdapter::AdaptReason adapt_reason_;
239 // The critical section to protect handling requests.
240 talk_base::CriticalSection request_critical_section_;
242 // The weighted average of cpu load over time. It's always updated (if cpu
243 // adaptation is on), but only used if cpu_smoothing_ is set.
244 float system_load_average_;
246 DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter);
249 } // namespace cricket
251 #endif // TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT