Backport of webrtc patch which is somehow missing. Not a true git patch, it's been hacked up manually. -gmt commit 7252a2ba8035c4128917a9558a3e34fc9dbe7c44 Author: Peter Boström Date: Mon May 18 19:42:03 2015 +0200 Add HW fallback option to software decoding. Permits falling back to software decoding for unsupported resolutions in bitstreams. BUG=4625, chromium:487934 R=mflodman@webrtc.org, stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/46269004 Cr-Commit-Position: refs/heads/master@{#9209} --- a/third_party/libjingle/source/talk/media/webrtc/webrtcvideoengine2.cc +++ b/third_party/libjingle/source/talk/media/webrtc/webrtcvideoengine2.cc @@ -2223,6 +2223,21 @@ WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( SetRecvCodecs(recv_codecs); } +WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder:: + AllocatedDecoder(webrtc::VideoDecoder* decoder, + webrtc::VideoCodecType type, + bool external) + : decoder(decoder), + external_decoder(nullptr), + type(type), + external(external) { + if (external) { + external_decoder = decoder; + this->decoder = + new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder); + } +} + WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() { call_->DestroyVideoReceiveStream(stream_); ClearDecoders(&allocated_decoders_); @@ -2330,10 +2345,9 @@ void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders( for (size_t i = 0; i < allocated_decoders->size(); ++i) { if ((*allocated_decoders)[i].external) { external_decoder_factory_->DestroyVideoDecoder( - (*allocated_decoders)[i].decoder); - } else { - delete (*allocated_decoders)[i].decoder; + (*allocated_decoders)[i].external_decoder); } + delete (*allocated_decoders)[i].decoder; } allocated_decoders->clear(); } --- a/third_party/libjingle/source/talk/media/webrtc/webrtcvideoengine2.h +++ b/third_party/libjungle/source/talk/media/webrtc/webrtcvideoengine2.h @@ -430,9 +430,10 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, struct AllocatedDecoder { AllocatedDecoder(webrtc::VideoDecoder* decoder, webrtc::VideoCodecType type, - bool external) - : decoder(decoder), type(type), external(external) {} + bool external); webrtc::VideoDecoder* decoder; + // Decoder wrapped into a fallback decoder to permit software fallback. + webrtc::VideoDecoder* external_decoder; webrtc::VideoCodecType type; bool external; }; --- a/third_party/webrtc/modules/video_coding/codecs/interface/video_error_codes.h +++ b/third_party/webrtc/modules/video_coding/codecs/interface/video_error_codes.h @@ -26,5 +26,6 @@ #define WEBRTC_VIDEO_CODEC_TIMEOUT -6 #define WEBRTC_VIDEO_CODEC_UNINITIALIZED -7 #define WEBRTC_VIDEO_CODEC_ERR_REQUEST_SLI -12 +#define WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE -13 #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_ERROR_CODES_H --- a/third_party/webrtc/video/BUILD.gn +++ b/third_party/webrtc/video/BUILD.gn @@ -21,6 +21,7 @@ source_set("video") { "send_statistics_proxy.h", "transport_adapter.cc", "transport_adapter.h", + "video_decoder.cc", "video_receive_stream.cc", "video_receive_stream.h", "video_send_stream.cc", --- a/third_party/webrtc/video/call.cc +++ b/third_party/webrtc/video/call.cc @@ -47,17 +47,6 @@ VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) { return nullptr; } -VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) { - switch (codec_type) { - case kVp8: - return VP8Decoder::Create(); - case kVp9: - return VP9Decoder::Create(); - } - RTC_NOTREACHED(); - return nullptr; -} - const int Call::Config::kDefaultStartBitrateBps = 300000; namespace internal { --- /dev/null +++ b/third_party/webrtc/video/video_decoder.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/video_decoder.h" + +#include "webrtc/base/checks.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" +#include "webrtc/system_wrappers/interface/logging.h" + +namespace webrtc { +VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) { + switch (codec_type) { + case kVp8: + return VP8Decoder::Create(); + case kVp9: + return VP9Decoder::Create(); + case kUnsupportedCodec: + RTC_NOTREACHED(); + return nullptr; + } + RTC_NOTREACHED(); + return nullptr; +} + +VideoDecoder::DecoderType CodecTypeToDecoderType(VideoCodecType codec_type) { + switch (codec_type) { + case kVideoCodecVP8: + return VideoDecoder::kVp8; + case kVideoCodecVP9: + return VideoDecoder::kVp9; + default: + return VideoDecoder::kUnsupportedCodec; + } +} + +VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper( + VideoCodecType codec_type, + VideoDecoder* decoder) + : decoder_type_(CodecTypeToDecoderType(codec_type)), + decoder_(decoder), + callback_(nullptr) { +} + +int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode( + const VideoCodec* codec_settings, + int32_t number_of_cores) { + codec_settings_ = *codec_settings; + number_of_cores_ = number_of_cores; + return decoder_->InitDecode(codec_settings, number_of_cores); +} + +bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() { + CHECK(decoder_type_ != kUnsupportedCodec) + << "Decoder requesting fallback to codec not supported in software."; + LOG(LS_WARNING) << "Decoder falling back to software decoding."; + fallback_decoder_.reset(VideoDecoder::Create(decoder_type_)); + if (fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_) != + WEBRTC_VIDEO_CODEC_OK) { + LOG(LS_ERROR) << "Failed to initialize software-decoder fallback."; + fallback_decoder_.reset(); + return false; + } + if (callback_ != nullptr) + fallback_decoder_->RegisterDecodeCompleteCallback(callback_); + return true; +} + +int32_t VideoDecoderSoftwareFallbackWrapper::Decode( + const EncodedImage& input_image, + bool missing_frames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) { + // Try decoding with the provided decoder on every keyframe or when there's no + // fallback decoder. This is the normal case. + if (!fallback_decoder_ || input_image._frameType == kKeyFrame) { + int32_t ret = decoder_->Decode(input_image, missing_frames, fragmentation, + codec_specific_info, render_time_ms); + if (ret == WEBRTC_VIDEO_CODEC_OK) { + if (fallback_decoder_) { + // Decode OK -> stop using fallback decoder. + fallback_decoder_->Release(); + fallback_decoder_.reset(); + return WEBRTC_VIDEO_CODEC_OK; + } + } + if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) + return ret; + if (!fallback_decoder_) { + // Try to initialize fallback decoder. + if (!InitFallbackDecoder()) + return ret; + } + } + return fallback_decoder_->Decode(input_image, missing_frames, fragmentation, + codec_specific_info, render_time_ms); +} + +int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + callback_ = callback; + int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback); + if (fallback_decoder_) + return fallback_decoder_->RegisterDecodeCompleteCallback(callback); + return ret; +} + +int32_t VideoDecoderSoftwareFallbackWrapper::Release() { + if (fallback_decoder_) + fallback_decoder_->Release(); + return decoder_->Release(); +} + +int32_t VideoDecoderSoftwareFallbackWrapper::Reset() { + if (fallback_decoder_) + fallback_decoder_->Reset(); + return decoder_->Reset(); +} + +} // namespace webrtc --- /dev/null +++ b/third_party/webrtc/video/video_decoder_unittest.cc @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/video_decoder.h" + +#include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h" + +namespace webrtc { + +class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test { + protected: + VideoDecoderSoftwareFallbackWrapperTest() + : fallback_wrapper_(kVideoCodecVP8, &fake_decoder_) {} + + class CountingFakeDecoder : public VideoDecoder { + public: + int32_t InitDecode(const VideoCodec* codec_settings, + int32_t number_of_cores) override { + ++init_decode_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override { + ++decode_count_; + return decode_return_code_; + } + + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override { + decode_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { + ++release_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Reset() override { + ++reset_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + int init_decode_count_ = 0; + int decode_count_ = 0; + int32_t decode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + DecodedImageCallback* decode_complete_callback_ = nullptr; + int release_count_ = 0; + int reset_count_ = 0; + }; + CountingFakeDecoder fake_decoder_; + VideoDecoderSoftwareFallbackWrapper fallback_wrapper_; +}; + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, InitializesDecoder) { + VideoCodec codec = {}; + fallback_wrapper_.InitDecode(&codec, 2); + EXPECT_EQ(1, fake_decoder_.init_decode_count_); +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, + CanRecoverFromSoftwareFallback) { + VideoCodec codec = {}; + fallback_wrapper_.InitDecode(&codec, 2); + // Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW + // decoder. + fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + EncodedImage encoded_image; + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); + EXPECT_EQ(1, fake_decoder_.decode_count_); + + // Fail -> fake_decoder shouldn't be used anymore. + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); + EXPECT_EQ(1, fake_decoder_.decode_count_) + << "Decoder used even though fallback should be active."; + + // Should be able to recover on a keyframe. + encoded_image._frameType = kKeyFrame; + fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); + EXPECT_EQ(2, fake_decoder_.decode_count_) + << "Wrapper did not try to decode a keyframe using registered decoder."; + + encoded_image._frameType = kDeltaFrame; + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); + EXPECT_EQ(3, fake_decoder_.decode_count_) + << "Decoder not used on future delta frames."; +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, DoesNotFallbackOnEveryError) { + VideoCodec codec = {}; + fallback_wrapper_.InitDecode(&codec, 2); + fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; + EncodedImage encoded_image; + EXPECT_EQ( + fake_decoder_.decode_return_code_, + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1)); + EXPECT_EQ(1, fake_decoder_.decode_count_); + + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); + EXPECT_EQ(2, fake_decoder_.decode_count_) + << "Decoder should be active even though previous decode failed."; +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsReleaseCall) { + VideoCodec codec = {}; + fallback_wrapper_.InitDecode(&codec, 2); + fallback_wrapper_.Release(); + EXPECT_EQ(1, fake_decoder_.release_count_); + + fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + EncodedImage encoded_image; + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); + EXPECT_EQ(1, fake_decoder_.release_count_) + << "Decoder should not be released during fallback."; + fallback_wrapper_.Release(); + EXPECT_EQ(2, fake_decoder_.release_count_); +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsResetCall) { + VideoCodec codec = {}; + fallback_wrapper_.InitDecode(&codec, 2); + fallback_wrapper_.Reset(); + EXPECT_EQ(1, fake_decoder_.reset_count_); + + fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + EncodedImage encoded_image; + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); + fallback_wrapper_.Reset(); + EXPECT_EQ(2, fake_decoder_.reset_count_) + << "Reset not forwarded during fallback."; +} + +// TODO(pbos): Fake a VP8 frame well enough to actually receive a callback from +// the software encoder. +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, + ForwardsRegisterDecodeCompleteCallback) { + class FakeDecodedImageCallback : public DecodedImageCallback { + int32_t Decoded(I420VideoFrame& decodedImage) override { return 0; } + } callback, callback2; + + VideoCodec codec = {}; + fallback_wrapper_.InitDecode(&codec, 2); + fallback_wrapper_.RegisterDecodeCompleteCallback(&callback); + EXPECT_EQ(&callback, fake_decoder_.decode_complete_callback_); + + fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + EncodedImage encoded_image; + fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1); + fallback_wrapper_.RegisterDecodeCompleteCallback(&callback2); + EXPECT_EQ(&callback2, fake_decoder_.decode_complete_callback_); +} + +} // namespace webrtc --- a/third_party/webrtc/video/webrtc_video.gypi +++ b/third_party/webrtc/video/webrtc_video.gypi @@ -22,6 +22,7 @@ 'video/receive_statistics_proxy.h', 'video/transport_adapter.cc', 'video/transport_adapter.h', + 'video/video_decoder.cc', 'video/video_receive_stream.cc', 'video/video_receive_stream.h', 'video/video_send_stream.cc', --- a/third_party/webrtc/video_decoder.h +++ b/third_party/webrtc/video_decoder.h @@ -40,21 +40,22 @@ class VideoDecoder { public: enum DecoderType { kVp8, - kVp9 + kVp9, + kUnsupportedCodec, }; static VideoDecoder* Create(DecoderType codec_type); virtual ~VideoDecoder() {} - virtual int32_t InitDecode(const VideoCodec* codecSettings, - int32_t numberOfCores) = 0; + virtual int32_t InitDecode(const VideoCodec* codec_settings, + int32_t number_of_cores) = 0; - virtual int32_t Decode(const EncodedImage& inputImage, - bool missingFrames, + virtual int32_t Decode(const EncodedImage& input_image, + bool missing_frames, const RTPFragmentationHeader* fragmentation, - const CodecSpecificInfo* codecSpecificInfo = NULL, - int64_t renderTimeMs = -1) = 0; + const CodecSpecificInfo* codec_specific_info = NULL, + int64_t render_time_ms = -1) = 0; virtual int32_t RegisterDecodeCompleteCallback( DecodedImageCallback* callback) = 0; @@ -70,6 +71,41 @@ class VideoDecoder { virtual VideoDecoder* Copy() { return NULL; } }; +// Class used to wrap external VideoDecoders to provide a fallback option on +// software decoding when a hardware decoder fails to decode a stream due to +// hardware restrictions, such as max resolution. +class VideoDecoderSoftwareFallbackWrapper : public webrtc::VideoDecoder { + public: + VideoDecoderSoftwareFallbackWrapper(VideoCodecType codec_type, + VideoDecoder* decoder); + + int32_t InitDecode(const VideoCodec* codec_settings, + int32_t number_of_cores) override; + + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + const RTPFragmentationHeader* fragmentation, + const CodecSpecificInfo* codec_specific_info, + int64_t render_time_ms) override; + + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + + int32_t Release() override; + int32_t Reset() override; + + private: + bool InitFallbackDecoder(); + + const DecoderType decoder_type_; + VideoDecoder* const decoder_; + + VideoCodec codec_settings_; + int32_t number_of_cores_; + rtc::scoped_ptr fallback_decoder_; + DecodedImageCallback* callback_; +}; + } // namespace webrtc #endif // WEBRTC_VIDEO_DECODER_H_ --- a/third_party/webrtc/webrtc_tests.gypi +++ b/third_party/webrtc/webrtc_tests.gypi @@ -148,6 +148,7 @@ 'video/bitrate_estimator_tests.cc', 'video/end_to_end_tests.cc', 'video/send_statistics_proxy_unittest.cc', + 'video/video_decoder_unittest.cc', 'video/video_send_stream_tests.cc', ], 'dependencies': [