| // libjingle |
| // Copyright 2010 Google Inc. |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are met: |
| // |
| // 1. Redistributions of source code must retain the above copyright notice, |
| // this list of conditions and the following disclaimer. |
| // 2. Redistributions in binary form must reproduce the above copyright notice, |
| // this list of conditions and the following disclaimer in the documentation |
| // and/or other materials provided with the distribution. |
| // 3. The name of the author may not be used to endorse or promote products |
| // derived from this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED |
| // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
| // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
| // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
| // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
| // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
| // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| #include "talk/session/phone/videoadapter.h" |
| |
| #include <limits.h> |
| |
| #include "talk/base/logging.h" |
| #include "talk/base/timeutils.h" |
| #include "talk/session/phone/videoframe.h" |
| |
| namespace cricket { |
| |
| // TODO: Make downgrades settable |
| static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU. |
| static const int kDefaultDowngradeWaitTimeMs = 2000; |
| |
| // Cpu system load thresholds relative to max cpus. |
| static const float kHighSystemThreshold = 0.95f; |
| static const float kLowSystemThreshold = 0.75f; |
| |
| // Cpu process load thresholds relative to current cpus. |
| static const float kMediumProcessThreshold = 0.50f; |
| |
| // TODO: Consider making scale factor table settable, to allow |
| // application to select quality vs performance tradeoff. |
| // List of scale factors that adapter will scale by. |
| #if defined(IOS) || defined(ANDROID) |
| // Mobile needs 1/4 scale for VGA (640x360) to QQVGA (160x90) |
| // or 1/4 scale for HVGA (480x270) to QQHVGA (120x67) |
| static const int kMinNumPixels = 120 * 67; |
| static float kScaleFactors[] = { |
| 1.f, // full size |
| 3.f/4.f, // 3/4 scale |
| 1.f/2.f, // 1/2 scale |
| 3.f/8.f, // 3/8 scale |
| 1.f/4.f, // 1/4 scale |
| }; |
| #else |
| // PC needs 1/8 scale for HD (1280x720) to QQVGA (160x90) |
| static const int kMinNumPixels = 160 * 100; |
| static float kScaleFactors[] = { |
| 1.f, // full size |
| 3.f/4.f, // 3/4 scale |
| 1.f/2.f, // 1/2 scale |
| 3.f/8.f, // 3/8 scale |
| 1.f/4.f, // 1/4 scale |
| 3.f/16.f, // 3/16 scale |
| 1.f/8.f // 1/8 scale |
| }; |
| #endif |
| |
| // Find scale factor that applied to width and height, is best match |
| // to num_pixels. |
| float VideoAdapter::FindClosestScale(int width, int height, |
| int target_num_pixels) { |
| if (!target_num_pixels) { |
| return 0.f; |
| } |
| int best_distance = INT_MAX; |
| int best_index = 0; // default to unscaled |
| for (size_t i = 0u; i < ARRAY_SIZE(kScaleFactors); ++i) { |
| int test_num_pixels = static_cast<int>(width * kScaleFactors[i] * |
| height * kScaleFactors[i]); |
| int diff = test_num_pixels - target_num_pixels; |
| if (diff < 0) { |
| diff = -diff; |
| } |
| if (diff < best_distance) { |
| best_distance = diff; |
| best_index = i; |
| if (!best_distance) { // Found exact match |
| break; |
| } |
| } |
| } |
| return kScaleFactors[best_index]; |
| } |
| |
| // There are several frame sizes used by Adapter. This explains them |
| // input_format - set once by server to frame size expected from the camera. |
| // output_format - size that output would like to be. Includes framerate. |
| // output_num_pixels - size that output should be constrained to. Used to |
| // compute output_format from in_frame. |
| // in_frame - actual camera captured frame size, which is typically the same |
| // as input_format. This can also be rotated or cropped for aspect ratio. |
| // out_frame - actual frame output by adapter. Should be a direct scale of |
| // in_frame maintaining rotation and aspect ratio. |
| // OnOutputFormatRequest - server requests you send this resolution based on |
| // view requests. |
| // OnEncoderResolutionRequest - encoder requests you send this resolution based |
| // on bandwidth |
| // OnCpuLoadUpdated - cpu monitor requests you send this resolution based on |
| // cpu load. |
| |
| /////////////////////////////////////////////////////////////////////// |
| // Implementation of VideoAdapter |
| VideoAdapter::VideoAdapter() |
| : output_num_pixels_(0), |
| black_output_(false), |
| is_black_(false), |
| drop_frame_count_(0) { |
| } |
| |
| VideoAdapter::~VideoAdapter() { |
| } |
| |
| // TODO: Consider SetInputFormat and SetOutputFormat without |
| // VideoFormat. |
| void VideoAdapter::SetInputFormat(const VideoFormat& format) { |
| talk_base::CritScope cs(&critical_section_); |
| input_format_ = format; |
| output_format_.interval = talk_base::_max( |
| output_format_.interval, input_format_.interval); |
| } |
| |
| void VideoAdapter::SetOutputFormat(const VideoFormat& format) { |
| talk_base::CritScope cs(&critical_section_); |
| output_format_ = format; |
| output_num_pixels_ = output_format_.width * output_format_.height; |
| output_format_.interval = talk_base::_max( |
| output_format_.interval, input_format_.interval); |
| drop_frame_count_ = 0; |
| } |
| |
| const VideoFormat& VideoAdapter::input_format() { |
| talk_base::CritScope cs(&critical_section_); |
| return input_format_; |
| } |
| |
| const VideoFormat& VideoAdapter::output_format() { |
| talk_base::CritScope cs(&critical_section_); |
| return output_format_; |
| } |
| |
| void VideoAdapter::SetBlackOutput(bool black) { |
| talk_base::CritScope cs(&critical_section_); |
| black_output_ = black; |
| } |
| |
| // Constrain output resolution to this many pixels overall |
| void VideoAdapter::SetOutputNumPixels(int num_pixels) { |
| output_num_pixels_ = num_pixels; |
| } |
| |
| int VideoAdapter::GetOutputNumPixels() const { |
| return output_num_pixels_; |
| } |
| |
| bool VideoAdapter::AdaptFrame(const VideoFrame* in_frame, |
| const VideoFrame** out_frame) { |
| talk_base::CritScope cs(&critical_section_); |
| |
| if (!in_frame || !out_frame || input_format_.IsSize0x0()) { |
| return false; |
| } |
| |
| // Drop the input frame if necessary. |
| bool should_drop = false; |
| if (!output_num_pixels_) { |
| // Drop all frames as the output format is 0x0. |
| should_drop = true; |
| } else { |
| // Drop some frames based on the ratio of the input fps and the output fps. |
| // We assume that the output fps is a factor of the input fps. In other |
| // words, the output interval is divided by the input interval evenly. |
| should_drop = (drop_frame_count_ > 0); |
| if (input_format_.interval > 0 && |
| output_format_.interval > input_format_.interval) { |
| ++drop_frame_count_; |
| drop_frame_count_ %= output_format_.interval / input_format_.interval; |
| } |
| } |
| |
| if (output_num_pixels_) { |
| float scale = VideoAdapter::FindClosestScale(in_frame->GetWidth(), |
| in_frame->GetHeight(), |
| output_num_pixels_); |
| output_format_.width = static_cast<int>(in_frame->GetWidth() * scale); |
| output_format_.height = static_cast<int>(in_frame->GetHeight() * scale); |
| } |
| |
| if (should_drop) { |
| *out_frame = NULL; |
| return true; |
| } |
| |
| if (!StretchToOutputFrame(in_frame)) { |
| return false; |
| } |
| |
| *out_frame = output_frame_.get(); |
| return true; |
| } |
| |
| bool VideoAdapter::StretchToOutputFrame(const VideoFrame* in_frame) { |
| int output_width = output_format_.width; |
| int output_height = output_format_.height; |
| |
| // Create and stretch the output frame if it has not been created yet or its |
| // size is not same as the expected. |
| bool stretched = false; |
| if (!output_frame_.get() || |
| output_frame_->GetWidth() != static_cast<size_t>(output_width) || |
| output_frame_->GetHeight() != static_cast<size_t>(output_height)) { |
| output_frame_.reset( |
| in_frame->Stretch(output_width, output_height, true, true)); |
| if (!output_frame_.get()) { |
| LOG(LS_WARNING) << "Adapter failed to stretch frame to " |
| << output_width << "x" << output_height; |
| return false; |
| } |
| stretched = true; |
| is_black_ = false; |
| } |
| |
| if (!black_output_) { |
| if (!stretched) { |
| // The output frame does not need to be blacken and has not been stretched |
| // from the input frame yet, stretch the input frame. This is the most |
| // common case. |
| in_frame->StretchToFrame(output_frame_.get(), true, true); |
| } |
| is_black_ = false; |
| } else { |
| if (!is_black_) { |
| output_frame_->SetToBlack(); |
| is_black_ = true; |
| } |
| output_frame_->SetElapsedTime(in_frame->GetElapsedTime()); |
| output_frame_->SetTimeStamp(in_frame->GetTimeStamp()); |
| } |
| |
| return true; |
| } |
| |
| /////////////////////////////////////////////////////////////////////// |
| // Implementation of CoordinatedVideoAdapter |
| CoordinatedVideoAdapter::CoordinatedVideoAdapter() |
| : cpu_adaptation_(false), |
| gd_adaptation_(true), |
| view_adaptation_(true), |
| cpu_downgrade_count_(0), |
| cpu_downgrade_wait_time_(0), |
| view_desired_num_pixels_(INT_MAX), |
| view_desired_interval_(0), |
| encoder_desired_num_pixels_(INT_MAX), |
| cpu_desired_num_pixels_(INT_MAX) { |
| } |
| |
| // Helper function to UPGRADE or DOWNGRADE a number of pixels |
| void CoordinatedVideoAdapter::StepPixelCount( |
| CoordinatedVideoAdapter::AdaptRequest request, |
| int* num_pixels) { |
| switch (request) { |
| case CoordinatedVideoAdapter::DOWNGRADE: |
| *num_pixels /= 2; |
| break; |
| |
| case CoordinatedVideoAdapter::UPGRADE: |
| *num_pixels *= 2; |
| break; |
| |
| default: // No change in pixel count |
| break; |
| } |
| return; |
| } |
| |
| // Find the adaptation request of the cpu based on the load. Return UPGRADE if |
| // the load is low, DOWNGRADE if the load is high, and KEEP otherwise. |
| CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest( |
| int current_cpus, int max_cpus, |
| float process_load, float system_load) { |
| // Downgrade if system is high and plugin is at least more than midrange. |
| if (system_load >= kHighSystemThreshold * max_cpus && |
| process_load >= kMediumProcessThreshold * current_cpus) { |
| return CoordinatedVideoAdapter::DOWNGRADE; |
| // Upgrade if system is low. |
| } else if (system_load < kLowSystemThreshold * max_cpus) { |
| return CoordinatedVideoAdapter::UPGRADE; |
| } |
| return CoordinatedVideoAdapter::KEEP; |
| } |
| |
| // A remote view request for a new resolution. |
| void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { |
| talk_base::CritScope cs(&request_critical_section_); |
| if (!view_adaptation_) { |
| return; |
| } |
| // Set output for initial aspect ratio in mediachannel unittests. |
| int old_num_pixels = GetOutputNumPixels(); |
| SetOutputFormat(format); |
| SetOutputNumPixels(old_num_pixels); |
| view_desired_num_pixels_ = format.width * format.height; |
| view_desired_interval_ = format.interval; |
| bool changed = AdaptToMinimumFormat(); |
| LOG(LS_INFO) << "VAdapt View Request: " |
| << format.width << "x" << format.height |
| << " Pixels: " << view_desired_num_pixels_ |
| << " Changed: " << (changed ? "true" : "false"); |
| } |
| |
| // A Bandwidth GD request for new resolution |
| void CoordinatedVideoAdapter::OnEncoderResolutionRequest( |
| int width, int height, AdaptRequest request) { |
| talk_base::CritScope cs(&request_critical_section_); |
| if (!gd_adaptation_) { |
| return; |
| } |
| if (KEEP != request) { |
| int new_encoder_desired_num_pixels = width * height; |
| int old_num_pixels = GetOutputNumPixels(); |
| if (new_encoder_desired_num_pixels != old_num_pixels) { |
| LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored"; |
| } else { |
| // Update the encoder desired format based on the request. |
| encoder_desired_num_pixels_ = new_encoder_desired_num_pixels; |
| StepPixelCount(request, &encoder_desired_num_pixels_); |
| } |
| } |
| bool changed = AdaptToMinimumFormat(); |
| LOG(LS_INFO) << "VAdapt GD Request: " |
| << (DOWNGRADE == request ? "down" : |
| (UPGRADE == request ? "up" : "keep")) |
| << " From: " << width << "x" << height |
| << " Pixels: " << encoder_desired_num_pixels_ |
| << " Changed: " << (changed ? "true" : "false"); |
| } |
| |
| // A CPU request for new resolution |
| void CoordinatedVideoAdapter::OnCpuLoadUpdated( |
| int current_cpus, int max_cpus, float process_load, float system_load) { |
| talk_base::CritScope cs(&request_critical_section_); |
| if (!cpu_adaptation_) { |
| return; |
| } |
| AdaptRequest request = FindCpuRequest(current_cpus, max_cpus, |
| process_load, system_load); |
| // Update how many times we have downgraded due to the cpu load. |
| switch (request) { |
| case DOWNGRADE: |
| if (cpu_downgrade_count_ < kMaxCpuDowngrades) { |
| // Ignore downgrades if we have downgraded the maximum times or we just |
| // downgraded in a short time. |
| if (cpu_downgrade_wait_time_ != 0 && |
| talk_base::TimeIsLater(talk_base::Time(), |
| cpu_downgrade_wait_time_)) { |
| LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade until " |
| << talk_base::TimeUntil(cpu_downgrade_wait_time_) |
| << " ms."; |
| request = KEEP; |
| } else { |
| ++cpu_downgrade_count_; |
| } |
| } else { |
| LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade " |
| "because maximum downgrades reached"; |
| } |
| break; |
| case UPGRADE: |
| if (cpu_downgrade_count_ > 0) { |
| bool is_min = IsMinimumFormat(cpu_desired_num_pixels_); |
| if (is_min) { |
| --cpu_downgrade_count_; |
| } else { |
| LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade " |
| "because cpu is not limiting resolution"; |
| } |
| } else { |
| LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade " |
| "because minimum downgrades reached"; |
| } |
| break; |
| case KEEP: |
| default: |
| break; |
| } |
| if (KEEP != request) { |
| // TODO: compute stepping up/down from OutputNumPixels but |
| // clamp to inputpixels / 4 (2 steps) |
| cpu_desired_num_pixels_ = static_cast<int>( |
| input_format().width * input_format().height >> cpu_downgrade_count_); |
| } |
| bool changed = AdaptToMinimumFormat(); |
| LOG(LS_INFO) << "VAdapt CPU Request: " |
| << (DOWNGRADE == request ? "down" : |
| (UPGRADE == request ? "up" : "keep")) |
| << " Process: " << process_load |
| << " System: " << system_load |
| << " Steps: " << cpu_downgrade_count_ |
| << " Changed: " << (changed ? "true" : "false"); |
| } |
| |
| // Called by cpu adapter on up requests. |
| bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) { |
| // Find closest scale factor that matches input resolution to min_num_pixels |
| // and set that for output resolution. This is not needed for VideoAdapter, |
| // but provides feedback to unittests and users on expected resolution. |
| // Actual resolution is based on input frame. |
| VideoFormat new_output = output_format(); |
| VideoFormat input = input_format(); |
| if (input_format().IsSize0x0()) { |
| input = new_output; |
| } |
| float scale = 1.0f; |
| if (!input.IsSize0x0()) { |
| scale = FindClosestScale(input.width, |
| input.height, |
| pixels); |
| } |
| new_output.width = static_cast<int>(input.width * scale); |
| new_output.height = static_cast<int>(input.height * scale); |
| int new_pixels = new_output.width * new_output.height; |
| int num_pixels = GetOutputNumPixels(); |
| return new_pixels <= num_pixels; |
| } |
| |
| // Called by all coordinators when there is a change. |
| bool CoordinatedVideoAdapter::AdaptToMinimumFormat() { |
| int old_num_pixels = GetOutputNumPixels(); |
| // Get the min of the formats that the server, encoder, and cpu wants. |
| int min_num_pixels = view_desired_num_pixels_; |
| if (encoder_desired_num_pixels_ && |
| (encoder_desired_num_pixels_ < min_num_pixels)) { |
| min_num_pixels = encoder_desired_num_pixels_; |
| } |
| if (cpu_adaptation_ && cpu_desired_num_pixels_ && |
| (cpu_desired_num_pixels_ < min_num_pixels)) { |
| min_num_pixels = cpu_desired_num_pixels_; |
| // Update the cpu_downgrade_wait_time_ if we are going to downgrade video. |
| cpu_downgrade_wait_time_ = |
| talk_base::TimeAfter(kDefaultDowngradeWaitTimeMs); |
| } |
| // prevent going below QQVGA |
| if (min_num_pixels > 0 && min_num_pixels < kMinNumPixels) { |
| min_num_pixels = kMinNumPixels; |
| } |
| SetOutputNumPixels(min_num_pixels); |
| |
| // Find closest scale factor that matches input resolution to min_num_pixels |
| // and set that for output resolution. This is not needed for VideoAdapter, |
| // but provides feedback to unittests and users on expected resolution. |
| // Actual resolution is based on input frame. |
| VideoFormat new_output = output_format(); |
| VideoFormat input = input_format(); |
| if (input_format().IsSize0x0()) { |
| input = new_output; |
| } |
| float scale = 1.0f; |
| if (!input.IsSize0x0()) { |
| scale = FindClosestScale(input.width, |
| input.height, |
| min_num_pixels); |
| } |
| new_output.width = static_cast<int>(input.width * scale); |
| new_output.height = static_cast<int>(input.height * scale); |
| new_output.interval = view_desired_interval_; |
| SetOutputFormat(new_output); |
| int new_num_pixels = GetOutputNumPixels(); |
| bool changed = new_num_pixels != old_num_pixels; |
| |
| LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_ |
| << " GD: " << encoder_desired_num_pixels_ |
| << " CPU: " << cpu_desired_num_pixels_ |
| << " Pixels: " << min_num_pixels |
| << " Scale: " << scale |
| << " Resolution: " << new_output.width |
| << "x" << new_output.height |
| << " Changed: " << (changed ? "true" : "false"); |
| return changed; |
| } |
| |
| } // namespace cricket |