Attachment #405448 for bug #547630

View | Details | Raw Unified | Return to bug 547630 | Differences between
Collapse All | Expand All




gmt
   Add HW fallback option to software decoding.
   
   Permits falling back to software decoding for unsupported resolutions in
   bitstreams.
   
   BUG=4625, chromium:487934
   R=mflodman@webrtc.org, stefan@webrtc.org
   
   Review URL: https://webrtc-codereview.appspot.com/46269004
   
   Cr-Commit-Position: refs/heads/master@{#9209}
++ b/third_party/libjingle/source/talk/media/webrtc/webrtcvideoengine2.cc

  SetRecvCodecs(recv_codecs);
}

WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder::
    AllocatedDecoder(webrtc::VideoDecoder* decoder,
                     webrtc::VideoCodecType type,
                     bool external)
    : decoder(decoder),
      external_decoder(nullptr),
      type(type),
      external(external) {
  if (external) {
    external_decoder = decoder;
    this->decoder =
        new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder);
  }
}

WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
  call_->DestroyVideoReceiveStream(stream_);
  ClearDecoders(&allocated_decoders_);

  for (size_t i = 0; i < allocated_decoders->size(); ++i) {
    if ((*allocated_decoders)[i].external) {
      external_decoder_factory_->DestroyVideoDecoder(
          (*allocated_decoders)[i].external_decoder);
    } else {
      delete (*allocated_decoders)[i].decoder;
    }
    delete (*allocated_decoders)[i].decoder;
  }
  allocated_decoders->clear();
}
++ b/third_party/libjungle/source/talk/media/webrtc/webrtcvideoengine2.h

    struct AllocatedDecoder {
      AllocatedDecoder(webrtc::VideoDecoder* decoder,
                       webrtc::VideoCodecType type,
                       bool external);
          : decoder(decoder), type(type), external(external) {}
      webrtc::VideoDecoder* decoder;
      // Decoder wrapped into a fallback decoder to permit software fallback.
      webrtc::VideoDecoder* external_decoder;
      webrtc::VideoCodecType type;
      bool external;
    };
++ b/third_party/webrtc/modules/video_coding/codecs/interface/video_error_codes.h

#define WEBRTC_VIDEO_CODEC_TIMEOUT -6
#define WEBRTC_VIDEO_CODEC_UNINITIALIZED -7
#define WEBRTC_VIDEO_CODEC_ERR_REQUEST_SLI -12
#define WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE -13

#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_ERROR_CODES_H
++ b/third_party/webrtc/video/BUILD.gn

    "send_statistics_proxy.h",
    "transport_adapter.cc",
    "transport_adapter.h",
    "video_decoder.cc",
    "video_receive_stream.cc",
    "video_receive_stream.h",
    "video_send_stream.cc",
++ b/third_party/webrtc/video/call.cc

  return nullptr;
}

VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) {
  switch (codec_type) {
    case kVp8:
      return VP8Decoder::Create();
    case kVp9:
      return VP9Decoder::Create();
  }
  RTC_NOTREACHED();
  return nullptr;
}

const int Call::Config::kDefaultStartBitrateBps = 300000;

namespace internal {
++ b/third_party/webrtc/video/video_decoder.cc

/*
 *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/video_decoder.h"

#include "webrtc/base/checks.h"
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
#include "webrtc/system_wrappers/interface/logging.h"

namespace webrtc {
VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) {
  switch (codec_type) {
    case kVp8:
      return VP8Decoder::Create();
    case kVp9:
      return VP9Decoder::Create();
    case kUnsupportedCodec:
      RTC_NOTREACHED();
      return nullptr;
  }
  RTC_NOTREACHED();
  return nullptr;
}

VideoDecoder::DecoderType CodecTypeToDecoderType(VideoCodecType codec_type) {
  switch (codec_type) {
    case kVideoCodecVP8:
      return VideoDecoder::kVp8;
    case kVideoCodecVP9:
      return VideoDecoder::kVp9;
    default:
      return VideoDecoder::kUnsupportedCodec;
  }
}

VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
    VideoCodecType codec_type,
    VideoDecoder* decoder)
    : decoder_type_(CodecTypeToDecoderType(codec_type)),
      decoder_(decoder),
      callback_(nullptr) {
}

int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode(
    const VideoCodec* codec_settings,
    int32_t number_of_cores) {
  codec_settings_ = *codec_settings;
  number_of_cores_ = number_of_cores;
  return decoder_->InitDecode(codec_settings, number_of_cores);
}

bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() {
  CHECK(decoder_type_ != kUnsupportedCodec)
      << "Decoder requesting fallback to codec not supported in software.";
  LOG(LS_WARNING) << "Decoder falling back to software decoding.";
  fallback_decoder_.reset(VideoDecoder::Create(decoder_type_));
  if (fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_) !=
      WEBRTC_VIDEO_CODEC_OK) {
    LOG(LS_ERROR) << "Failed to initialize software-decoder fallback.";
    fallback_decoder_.reset();
    return false;
  }
  if (callback_ != nullptr)
    fallback_decoder_->RegisterDecodeCompleteCallback(callback_);
  return true;
}

int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
    const EncodedImage& input_image,
    bool missing_frames,
    const RTPFragmentationHeader* fragmentation,
    const CodecSpecificInfo* codec_specific_info,
    int64_t render_time_ms) {
  // Try decoding with the provided decoder on every keyframe or when there's no
  // fallback decoder. This is the normal case.
  if (!fallback_decoder_ || input_image._frameType == kKeyFrame) {
    int32_t ret = decoder_->Decode(input_image, missing_frames, fragmentation,
                                   codec_specific_info, render_time_ms);
    if (ret == WEBRTC_VIDEO_CODEC_OK) {
      if (fallback_decoder_) {
        // Decode OK -> stop using fallback decoder.
        fallback_decoder_->Release();
        fallback_decoder_.reset();
        return WEBRTC_VIDEO_CODEC_OK;
      }
    }
    if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)
      return ret;
    if (!fallback_decoder_) {
      // Try to initialize fallback decoder.
      if (!InitFallbackDecoder())
        return ret;
    }
  }
  return fallback_decoder_->Decode(input_image, missing_frames, fragmentation,
                                   codec_specific_info, render_time_ms);
}

int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback(
    DecodedImageCallback* callback) {
  callback_ = callback;
  int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
  if (fallback_decoder_)
    return fallback_decoder_->RegisterDecodeCompleteCallback(callback);
  return ret;
}

int32_t VideoDecoderSoftwareFallbackWrapper::Release() {
  if (fallback_decoder_)
    fallback_decoder_->Release();
  return decoder_->Release();
}

int32_t VideoDecoderSoftwareFallbackWrapper::Reset() {
  if (fallback_decoder_)
    fallback_decoder_->Reset();
  return decoder_->Reset();
}

}  // namespace webrtc
++ b/third_party/webrtc/video/video_decoder_unittest.cc

/*
 *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/video_decoder.h"

#include "testing/gtest/include/gtest/gtest.h"
#include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h"

namespace webrtc {

class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test {
 protected:
  VideoDecoderSoftwareFallbackWrapperTest()
      : fallback_wrapper_(kVideoCodecVP8, &fake_decoder_) {}

  class CountingFakeDecoder : public VideoDecoder {
   public:
    int32_t InitDecode(const VideoCodec* codec_settings,
                       int32_t number_of_cores) override {
      ++init_decode_count_;
      return WEBRTC_VIDEO_CODEC_OK;
    }

    int32_t Decode(const EncodedImage& input_image,
                   bool missing_frames,
                   const RTPFragmentationHeader* fragmentation,
                   const CodecSpecificInfo* codec_specific_info,
                   int64_t render_time_ms) override {
      ++decode_count_;
      return decode_return_code_;
    }

    int32_t RegisterDecodeCompleteCallback(
        DecodedImageCallback* callback) override {
      decode_complete_callback_ = callback;
      return WEBRTC_VIDEO_CODEC_OK;
    }

    int32_t Release() override {
      ++release_count_;
      return WEBRTC_VIDEO_CODEC_OK;
    }

    int32_t Reset() override {
      ++reset_count_;
      return WEBRTC_VIDEO_CODEC_OK;
    }
    int init_decode_count_ = 0;
    int decode_count_ = 0;
    int32_t decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
    DecodedImageCallback* decode_complete_callback_ = nullptr;
    int release_count_ = 0;
    int reset_count_ = 0;
  };
  CountingFakeDecoder fake_decoder_;
  VideoDecoderSoftwareFallbackWrapper fallback_wrapper_;
};

TEST_F(VideoDecoderSoftwareFallbackWrapperTest, InitializesDecoder) {
  VideoCodec codec = {};
  fallback_wrapper_.InitDecode(&codec, 2);
  EXPECT_EQ(1, fake_decoder_.init_decode_count_);
}

TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
       CanRecoverFromSoftwareFallback) {
  VideoCodec codec = {};
  fallback_wrapper_.InitDecode(&codec, 2);
  // Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW
  // decoder.
  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
  EncodedImage encoded_image;
  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
  EXPECT_EQ(1, fake_decoder_.decode_count_);

  // Fail -> fake_decoder shouldn't be used anymore.
  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
  EXPECT_EQ(1, fake_decoder_.decode_count_)
      << "Decoder used even though fallback should be active.";

  // Should be able to recover on a keyframe.
  encoded_image._frameType = kKeyFrame;
  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
  EXPECT_EQ(2, fake_decoder_.decode_count_)
      << "Wrapper did not try to decode a keyframe using registered decoder.";

  encoded_image._frameType = kDeltaFrame;
  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
  EXPECT_EQ(3, fake_decoder_.decode_count_)
      << "Decoder not used on future delta frames.";
}

TEST_F(VideoDecoderSoftwareFallbackWrapperTest, DoesNotFallbackOnEveryError) {
  VideoCodec codec = {};
  fallback_wrapper_.InitDecode(&codec, 2);
  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
  EncodedImage encoded_image;
  EXPECT_EQ(
      fake_decoder_.decode_return_code_,
      fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1));
  EXPECT_EQ(1, fake_decoder_.decode_count_);

  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
  EXPECT_EQ(2, fake_decoder_.decode_count_)
      << "Decoder should be active even though previous decode failed.";
}

TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsReleaseCall) {
  VideoCodec codec = {};
  fallback_wrapper_.InitDecode(&codec, 2);
  fallback_wrapper_.Release();
  EXPECT_EQ(1, fake_decoder_.release_count_);

  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
  EncodedImage encoded_image;
  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
  EXPECT_EQ(1, fake_decoder_.release_count_)
      << "Decoder should not be released during fallback.";
  fallback_wrapper_.Release();
  EXPECT_EQ(2, fake_decoder_.release_count_);
}

TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsResetCall) {
  VideoCodec codec = {};
  fallback_wrapper_.InitDecode(&codec, 2);
  fallback_wrapper_.Reset();
  EXPECT_EQ(1, fake_decoder_.reset_count_);

  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
  EncodedImage encoded_image;
  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
  fallback_wrapper_.Reset();
  EXPECT_EQ(2, fake_decoder_.reset_count_)
      << "Reset not forwarded during fallback.";
}

// TODO(pbos): Fake a VP8 frame well enough to actually receive a callback from
// the software encoder.
TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
       ForwardsRegisterDecodeCompleteCallback) {
  class FakeDecodedImageCallback : public DecodedImageCallback {
    int32_t Decoded(I420VideoFrame& decodedImage) override { return 0; }
  } callback, callback2;

  VideoCodec codec = {};
  fallback_wrapper_.InitDecode(&codec, 2);
  fallback_wrapper_.RegisterDecodeCompleteCallback(&callback);
  EXPECT_EQ(&callback, fake_decoder_.decode_complete_callback_);

  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
  EncodedImage encoded_image;
  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
  fallback_wrapper_.RegisterDecodeCompleteCallback(&callback2);
  EXPECT_EQ(&callback2, fake_decoder_.decode_complete_callback_);
}

}  // namespace webrtc
++ b/third_party/webrtc/video/webrtc_video.gypi

      'video/receive_statistics_proxy.h',
      'video/transport_adapter.cc',
      'video/transport_adapter.h',
      'video/video_decoder.cc',
      'video/video_receive_stream.cc',
      'video/video_receive_stream.h',
      'video/video_send_stream.cc',
++ b/third_party/webrtc/video_decoder.h

 public:
  enum DecoderType {
    kVp8,
    kVp9,
    kUnsupportedCodec,
  };

  static VideoDecoder* Create(DecoderType codec_type);

  virtual ~VideoDecoder() {}

  virtual int32_t InitDecode(const VideoCodec* codec_settings,
                             int32_t number_of_cores) = 0;

  virtual int32_t Decode(const EncodedImage& input_image,
                         bool missing_frames,
                         const RTPFragmentationHeader* fragmentation,
                         const CodecSpecificInfo* codec_specific_info = NULL,
                         int64_t render_time_ms = -1) = 0;

  virtual int32_t RegisterDecodeCompleteCallback(
      DecodedImageCallback* callback) = 0;

  virtual VideoDecoder* Copy() { return NULL; }
};

// Class used to wrap external VideoDecoders to provide a fallback option on
// software decoding when a hardware decoder fails to decode a stream due to
// hardware restrictions, such as max resolution.
class VideoDecoderSoftwareFallbackWrapper : public webrtc::VideoDecoder {
 public:
  VideoDecoderSoftwareFallbackWrapper(VideoCodecType codec_type,
                                      VideoDecoder* decoder);

  int32_t InitDecode(const VideoCodec* codec_settings,
                     int32_t number_of_cores) override;

  int32_t Decode(const EncodedImage& input_image,
                 bool missing_frames,
                 const RTPFragmentationHeader* fragmentation,
                 const CodecSpecificInfo* codec_specific_info,
                 int64_t render_time_ms) override;

  int32_t RegisterDecodeCompleteCallback(
      DecodedImageCallback* callback) override;

  int32_t Release() override;
  int32_t Reset() override;

 private:
  bool InitFallbackDecoder();

  const DecoderType decoder_type_;
  VideoDecoder* const decoder_;

  VideoCodec codec_settings_;
  int32_t number_of_cores_;
  rtc::scoped_ptr<VideoDecoder> fallback_decoder_;
  DecodedImageCallback* callback_;
};

}  // namespace webrtc

#endif  // WEBRTC_VIDEO_DECODER_H_
++ b/third_party/webrtc/webrtc_tests.gypi

        'video/bitrate_estimator_tests.cc',
        'video/end_to_end_tests.cc',
        'video/send_statistics_proxy_unittest.cc',
        'video/video_decoder_unittest.cc',
        'video/video_send_stream_tests.cc',
      ],
      'dependencies': [

Return to bug 547630

Line Link Here

(-)file_not_specified_in_diff (-34 / +370 lines)
	gmt
1	Add HW fallback option to software decoding.		Add HW fallback option to software decoding.
2		1
3	Permits falling back to software decoding for unsupported resolutions in	2	Permits falling back to software decoding for unsupported resolutions in
4	bitstreams.	3	bitstreams.
5		4
6	BUG=4625, chromium:487934	5	BUG=4625, chromium:487934
7	R=mflodman@webrtc.org, stefan@webrtc.org	6	R=mflodman@webrtc.org, stefan@webrtc.org
8		7
9	Review URL: https://webrtc-codereview.appspot.com/46269004	8	Review URL: https://webrtc-codereview.appspot.com/46269004
10		9
11	Cr-Commit-Position: refs/heads/master@{#9209}	10	Cr-Commit-Position: refs/heads/master@{#9209}
12	-- a/third_party/libjingle/source/talk/media/webrtc/webrtcvideoengine2.cc	11	++ b/third_party/libjingle/source/talk/media/webrtc/webrtcvideoengine2.cc
Lines 2223-2228 WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( Link Here
2223	SetRecvCodecs(recv_codecs);	2223	SetRecvCodecs(recv_codecs);
2224	}	2224	}
2225		2225
		2226	WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder::
		2227	AllocatedDecoder(webrtc::VideoDecoder* decoder,
		2228	webrtc::VideoCodecType type,
		2229	bool external)
		2230	: decoder(decoder),
		2231	external_decoder(nullptr),
		2232	type(type),
		2233	external(external) {
		2234	if (external) {
		2235	external_decoder = decoder;
		2236	this->decoder =
		2237	new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder);
		2238	}
		2239	}
		2240
2226	WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {	2241	WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
2227	call_->DestroyVideoReceiveStream(stream_);	2242	call_->DestroyVideoReceiveStream(stream_);
2228	ClearDecoders(&allocated_decoders_);	2243	ClearDecoders(&allocated_decoders_);
Lines 2330-2339 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders( Link Here
2330	for (size_t i = 0; i < allocated_decoders->size(); ++i) {	2345	for (size_t i = 0; i < allocated_decoders->size(); ++i) {
2331	if ((*allocated_decoders)[i].external) {	2346	if ((*allocated_decoders)[i].external) {
2332	external_decoder_factory_->DestroyVideoDecoder(	2347	external_decoder_factory_->DestroyVideoDecoder(
2333	(*allocated_decoders)[i].decoder);	2348	(*allocated_decoders)[i].external_decoder);
2334	} else {
2335	delete (*allocated_decoders)[i].decoder;
2336	}	2349	}
		2350	delete (*allocated_decoders)[i].decoder;
2337	}	2351	}
2338	allocated_decoders->clear();	2352	allocated_decoders->clear();
2339	}	2353	}
2340	-- a/third_party/libjingle/source/talk/media/webrtc/webrtcvideoengine2.h	2354	++ b/third_party/libjungle/source/talk/media/webrtc/webrtcvideoengine2.h
Lines 430-438 class WebRtcVideoChannel2 : public rtc::MessageHandler, Link Here
430	struct AllocatedDecoder {	430	struct AllocatedDecoder {
431	AllocatedDecoder(webrtc::VideoDecoder* decoder,	431	AllocatedDecoder(webrtc::VideoDecoder* decoder,
432	webrtc::VideoCodecType type,	432	webrtc::VideoCodecType type,
433	bool external)	433	bool external);
434	: decoder(decoder), type(type), external(external) {}
435	webrtc::VideoDecoder* decoder;	434	webrtc::VideoDecoder* decoder;
		435	// Decoder wrapped into a fallback decoder to permit software fallback.
		436	webrtc::VideoDecoder* external_decoder;
436	webrtc::VideoCodecType type;	437	webrtc::VideoCodecType type;
437	bool external;	438	bool external;
438	};	439	};
439	-- a/third_party/webrtc/modules/video_coding/codecs/interface/video_error_codes.h	440	++ b/third_party/webrtc/modules/video_coding/codecs/interface/video_error_codes.h
Lines 26-30 Link Here
26	#define WEBRTC_VIDEO_CODEC_TIMEOUT -6	26	#define WEBRTC_VIDEO_CODEC_TIMEOUT -6
27	#define WEBRTC_VIDEO_CODEC_UNINITIALIZED -7	27	#define WEBRTC_VIDEO_CODEC_UNINITIALIZED -7
28	#define WEBRTC_VIDEO_CODEC_ERR_REQUEST_SLI -12	28	#define WEBRTC_VIDEO_CODEC_ERR_REQUEST_SLI -12
		29	#define WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE -13
29		30
30	#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_ERROR_CODES_H	31	#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_ERROR_CODES_H
31	-- a/third_party/webrtc/video/BUILD.gn	32	++ b/third_party/webrtc/video/BUILD.gn
Lines 21-26 source_set("video") { Link Here
21	"send_statistics_proxy.h",	21	"send_statistics_proxy.h",
22	"transport_adapter.cc",	22	"transport_adapter.cc",
23	"transport_adapter.h",	23	"transport_adapter.h",
		24	"video_decoder.cc",
24	"video_receive_stream.cc",	25	"video_receive_stream.cc",
25	"video_receive_stream.h",	26	"video_receive_stream.h",
26	"video_send_stream.cc",	27	"video_send_stream.cc",
27	-- a/third_party/webrtc/video/call.cc	28	++ b/third_party/webrtc/video/call.cc
Lines 47-63 VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) { Link Here
47	return nullptr;	47	return nullptr;
48	}	48	}
49		49
50	VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) {
51	switch (codec_type) {
52	case kVp8:
53	return VP8Decoder::Create();
54	case kVp9:
55	return VP9Decoder::Create();
56	}
57	RTC_NOTREACHED();
58	return nullptr;
59	}
60
61	const int Call::Config::kDefaultStartBitrateBps = 300000;	50	const int Call::Config::kDefaultStartBitrateBps = 300000;
62		51
63	namespace internal {	52	namespace internal {
64	-- /dev/null	53	++ b/third_party/webrtc/video/video_decoder.cc
Line 0 Link Here
0	-- /dev/null	1	/*
		2	* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
		3	*
		4	* Use of this source code is governed by a BSD-style license
		5	* that can be found in the LICENSE file in the root of the source
		6	* tree. An additional intellectual property rights grant can be found
		7	* in the file PATENTS. All contributing project authors may
		8	* be found in the AUTHORS file in the root of the source tree.
		9	*/
		10
		11	#include "webrtc/video_decoder.h"
		12
		13	#include "webrtc/base/checks.h"
		14	#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
		15	#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
		16	#include "webrtc/system_wrappers/interface/logging.h"
		17
		18	namespace webrtc {
		19	VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) {
		20	switch (codec_type) {
		21	case kVp8:
		22	return VP8Decoder::Create();
		23	case kVp9:
		24	return VP9Decoder::Create();
		25	case kUnsupportedCodec:
		26	RTC_NOTREACHED();
		27	return nullptr;
		28	}
		29	RTC_NOTREACHED();
		30	return nullptr;
		31	}
		32
		33	VideoDecoder::DecoderType CodecTypeToDecoderType(VideoCodecType codec_type) {
		34	switch (codec_type) {
		35	case kVideoCodecVP8:
		36	return VideoDecoder::kVp8;
		37	case kVideoCodecVP9:
		38	return VideoDecoder::kVp9;
		39	default:
		40	return VideoDecoder::kUnsupportedCodec;
		41	}
		42	}
		43
		44	VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
		45	VideoCodecType codec_type,
		46	VideoDecoder* decoder)
		47	: decoder_type_(CodecTypeToDecoderType(codec_type)),
		48	decoder_(decoder),
		49	callback_(nullptr) {
		50	}
		51
		52	int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode(
		53	const VideoCodec* codec_settings,
		54	int32_t number_of_cores) {
		55	codec_settings_ = *codec_settings;
		56	number_of_cores_ = number_of_cores;
		57	return decoder_->InitDecode(codec_settings, number_of_cores);
		58	}
		59
		60	bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() {
		61	CHECK(decoder_type_ != kUnsupportedCodec)
		62	<< "Decoder requesting fallback to codec not supported in software.";
		63	LOG(LS_WARNING) << "Decoder falling back to software decoding.";
		64	fallback_decoder_.reset(VideoDecoder::Create(decoder_type_));
		65	if (fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_) !=
66	WEBRTC_VIDEO_CODEC_OK) {
67	LOG(LS_ERROR) << "Failed to initialize software-decoder fallback.";
68	fallback_decoder_.reset();
69	return false;
70	}
71	if (callback_ != nullptr)
72	fallback_decoder_->RegisterDecodeCompleteCallback(callback_);
73	return true;
74	}
75
76	int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
77	const EncodedImage& input_image,
78	bool missing_frames,
79	const RTPFragmentationHeader* fragmentation,
80	const CodecSpecificInfo* codec_specific_info,
81	int64_t render_time_ms) {
82	// Try decoding with the provided decoder on every keyframe or when there's no
83	// fallback decoder. This is the normal case.
84	if (!fallback_decoder_ \|\| input_image._frameType == kKeyFrame) {
85	int32_t ret = decoder_->Decode(input_image, missing_frames, fragmentation,
86	codec_specific_info, render_time_ms);
87	if (ret == WEBRTC_VIDEO_CODEC_OK) {
88	if (fallback_decoder_) {
89	// Decode OK -> stop using fallback decoder.
90	fallback_decoder_->Release();
91	fallback_decoder_.reset();
92	return WEBRTC_VIDEO_CODEC_OK;
93	}
94	}
95	if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)
96	return ret;
97	if (!fallback_decoder_) {
98	// Try to initialize fallback decoder.
99	if (!InitFallbackDecoder())
100	return ret;
101	}
102	}
103	return fallback_decoder_->Decode(input_image, missing_frames, fragmentation,
104	codec_specific_info, render_time_ms);
105	}
106
107	int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback(
108	DecodedImageCallback* callback) {
109	callback_ = callback;
110	int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
111	if (fallback_decoder_)
112	return fallback_decoder_->RegisterDecodeCompleteCallback(callback);
113	return ret;
114	}
115
116	int32_t VideoDecoderSoftwareFallbackWrapper::Release() {
117	if (fallback_decoder_)
118	fallback_decoder_->Release();
119	return decoder_->Release();
120	}
121
122	int32_t VideoDecoderSoftwareFallbackWrapper::Reset() {
123	if (fallback_decoder_)
124	fallback_decoder_->Reset();
125	return decoder_->Reset();
126	}
127
128	} // namespace webrtc
		129	++ b/third_party/webrtc/video/video_decoder_unittest.cc
Line 0 Link Here
0	-- a/third_party/webrtc/video/webrtc_video.gypi	1	/*
		2	* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
		3	*
		4	* Use of this source code is governed by a BSD-style license
		5	* that can be found in the LICENSE file in the root of the source
		6	* tree. An additional intellectual property rights grant can be found
		7	* in the file PATENTS. All contributing project authors may
		8	* be found in the AUTHORS file in the root of the source tree.
		9	*/
		10
		11	#include "webrtc/video_decoder.h"
		12
		13	#include "testing/gtest/include/gtest/gtest.h"
		14	#include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h"
		15
		16	namespace webrtc {
		17
		18	class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test {
		19	protected:
		20	VideoDecoderSoftwareFallbackWrapperTest()
		21	: fallback_wrapper_(kVideoCodecVP8, &fake_decoder_) {}
		22
		23	class CountingFakeDecoder : public VideoDecoder {
		24	public:
		25	int32_t InitDecode(const VideoCodec* codec_settings,
		26	int32_t number_of_cores) override {
		27	++init_decode_count_;
		28	return WEBRTC_VIDEO_CODEC_OK;
		29	}
		30
		31	int32_t Decode(const EncodedImage& input_image,
		32	bool missing_frames,
		33	const RTPFragmentationHeader* fragmentation,
		34	const CodecSpecificInfo* codec_specific_info,
		35	int64_t render_time_ms) override {
		36	++decode_count_;
		37	return decode_return_code_;
		38	}
		39
		40	int32_t RegisterDecodeCompleteCallback(
		41	DecodedImageCallback* callback) override {
		42	decode_complete_callback_ = callback;
		43	return WEBRTC_VIDEO_CODEC_OK;
		44	}
		45
		46	int32_t Release() override {
		47	++release_count_;
		48	return WEBRTC_VIDEO_CODEC_OK;
		49	}
		50
		51	int32_t Reset() override {
		52	++reset_count_;
		53	return WEBRTC_VIDEO_CODEC_OK;
		54	}
		55	int init_decode_count_ = 0;
		56	int decode_count_ = 0;
		57	int32_t decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
		58	DecodedImageCallback* decode_complete_callback_ = nullptr;
		59	int release_count_ = 0;
		60	int reset_count_ = 0;
		61	};
		62	CountingFakeDecoder fake_decoder_;
		63	VideoDecoderSoftwareFallbackWrapper fallback_wrapper_;
		64	};
		65
66	TEST_F(VideoDecoderSoftwareFallbackWrapperTest, InitializesDecoder) {
67	VideoCodec codec = {};
68	fallback_wrapper_.InitDecode(&codec, 2);
69	EXPECT_EQ(1, fake_decoder_.init_decode_count_);
70	}
71
72	TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
73	CanRecoverFromSoftwareFallback) {
74	VideoCodec codec = {};
75	fallback_wrapper_.InitDecode(&codec, 2);
76	// Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW
77	// decoder.
78	fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
79	EncodedImage encoded_image;
80	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
81	EXPECT_EQ(1, fake_decoder_.decode_count_);
82
83	// Fail -> fake_decoder shouldn't be used anymore.
84	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
85	EXPECT_EQ(1, fake_decoder_.decode_count_)
86	<< "Decoder used even though fallback should be active.";
87
88	// Should be able to recover on a keyframe.
89	encoded_image._frameType = kKeyFrame;
90	fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
91	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
92	EXPECT_EQ(2, fake_decoder_.decode_count_)
93	<< "Wrapper did not try to decode a keyframe using registered decoder.";
94
95	encoded_image._frameType = kDeltaFrame;
96	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
97	EXPECT_EQ(3, fake_decoder_.decode_count_)
98	<< "Decoder not used on future delta frames.";
99	}
100
101	TEST_F(VideoDecoderSoftwareFallbackWrapperTest, DoesNotFallbackOnEveryError) {
102	VideoCodec codec = {};
103	fallback_wrapper_.InitDecode(&codec, 2);
104	fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
105	EncodedImage encoded_image;
106	EXPECT_EQ(
107	fake_decoder_.decode_return_code_,
108	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1));
109	EXPECT_EQ(1, fake_decoder_.decode_count_);
110
111	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
112	EXPECT_EQ(2, fake_decoder_.decode_count_)
113	<< "Decoder should be active even though previous decode failed.";
114	}
115
116	TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsReleaseCall) {
117	VideoCodec codec = {};
118	fallback_wrapper_.InitDecode(&codec, 2);
119	fallback_wrapper_.Release();
120	EXPECT_EQ(1, fake_decoder_.release_count_);
121
122	fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
123	EncodedImage encoded_image;
124	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
125	EXPECT_EQ(1, fake_decoder_.release_count_)
126	<< "Decoder should not be released during fallback.";
127	fallback_wrapper_.Release();
128	EXPECT_EQ(2, fake_decoder_.release_count_);
129	}
130
131	TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsResetCall) {
132	VideoCodec codec = {};
133	fallback_wrapper_.InitDecode(&codec, 2);
134	fallback_wrapper_.Reset();
135	EXPECT_EQ(1, fake_decoder_.reset_count_);
136
137	fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
138	EncodedImage encoded_image;
139	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
140	fallback_wrapper_.Reset();
141	EXPECT_EQ(2, fake_decoder_.reset_count_)
142	<< "Reset not forwarded during fallback.";
143	}
144
145	// TODO(pbos): Fake a VP8 frame well enough to actually receive a callback from
146	// the software encoder.
147	TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
148	ForwardsRegisterDecodeCompleteCallback) {
149	class FakeDecodedImageCallback : public DecodedImageCallback {
150	int32_t Decoded(I420VideoFrame& decodedImage) override { return 0; }
151	} callback, callback2;
152
153	VideoCodec codec = {};
154	fallback_wrapper_.InitDecode(&codec, 2);
155	fallback_wrapper_.RegisterDecodeCompleteCallback(&callback);
156	EXPECT_EQ(&callback, fake_decoder_.decode_complete_callback_);
157
158	fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
159	EncodedImage encoded_image;
160	fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
161	fallback_wrapper_.RegisterDecodeCompleteCallback(&callback2);
162	EXPECT_EQ(&callback2, fake_decoder_.decode_complete_callback_);
163	}
164
165	} // namespace webrtc
		166	++ b/third_party/webrtc/video/webrtc_video.gypi
Lines 22-27 Link Here
22	'video/receive_statistics_proxy.h',	22	'video/receive_statistics_proxy.h',
23	'video/transport_adapter.cc',	23	'video/transport_adapter.cc',
24	'video/transport_adapter.h',	24	'video/transport_adapter.h',
		25	'video/video_decoder.cc',
25	'video/video_receive_stream.cc',	26	'video/video_receive_stream.cc',
26	'video/video_receive_stream.h',	27	'video/video_receive_stream.h',
27	'video/video_send_stream.cc',	28	'video/video_send_stream.cc',
28	-- a/third_party/webrtc/video_decoder.h	29	++ b/third_party/webrtc/video_decoder.h
Lines 40-60 class VideoDecoder { Link Here
40	public:	40	public:
41	enum DecoderType {	41	enum DecoderType {
42	kVp8,	42	kVp8,
43	kVp9	43	kVp9,
		44	kUnsupportedCodec,
44	};	45	};
45		46
46	static VideoDecoder* Create(DecoderType codec_type);	47	static VideoDecoder* Create(DecoderType codec_type);
47		48
48	virtual ~VideoDecoder() {}	49	virtual ~VideoDecoder() {}
49		50
50	virtual int32_t InitDecode(const VideoCodec* codecSettings,	51	virtual int32_t InitDecode(const VideoCodec* codec_settings,
51	int32_t numberOfCores) = 0;	52	int32_t number_of_cores) = 0;
52		53
53	virtual int32_t Decode(const EncodedImage& inputImage,	54	virtual int32_t Decode(const EncodedImage& input_image,
54	bool missingFrames,	55	bool missing_frames,
55	const RTPFragmentationHeader* fragmentation,	56	const RTPFragmentationHeader* fragmentation,
56	const CodecSpecificInfo* codecSpecificInfo = NULL,	57	const CodecSpecificInfo* codec_specific_info = NULL,
57	int64_t renderTimeMs = -1) = 0;	58	int64_t render_time_ms = -1) = 0;
58		59
59	virtual int32_t RegisterDecodeCompleteCallback(	60	virtual int32_t RegisterDecodeCompleteCallback(
60	DecodedImageCallback* callback) = 0;	61	DecodedImageCallback* callback) = 0;
Lines 70-75 class VideoDecoder { Link Here
70	virtual VideoDecoder* Copy() { return NULL; }	71	virtual VideoDecoder* Copy() { return NULL; }
71	};	72	};
72		73
		74	// Class used to wrap external VideoDecoders to provide a fallback option on
		75	// software decoding when a hardware decoder fails to decode a stream due to
		76	// hardware restrictions, such as max resolution.
		77	class VideoDecoderSoftwareFallbackWrapper : public webrtc::VideoDecoder {
		78	public:
		79	VideoDecoderSoftwareFallbackWrapper(VideoCodecType codec_type,
		80	VideoDecoder* decoder);
		81
		82	int32_t InitDecode(const VideoCodec* codec_settings,
		83	int32_t number_of_cores) override;
		84
		85	int32_t Decode(const EncodedImage& input_image,
		86	bool missing_frames,
		87	const RTPFragmentationHeader* fragmentation,
		88	const CodecSpecificInfo* codec_specific_info,
		89	int64_t render_time_ms) override;
		90
		91	int32_t RegisterDecodeCompleteCallback(
		92	DecodedImageCallback* callback) override;
		93
		94	int32_t Release() override;
		95	int32_t Reset() override;
		96
		97	private:
		98	bool InitFallbackDecoder();
		99
		100	const DecoderType decoder_type_;
		101	VideoDecoder* const decoder_;
		102
		103	VideoCodec codec_settings_;
		104	int32_t number_of_cores_;
		105	rtc::scoped_ptr<VideoDecoder> fallback_decoder_;
		106	DecodedImageCallback* callback_;
		107	};
		108
73	} // namespace webrtc	109	} // namespace webrtc
74		110
75	#endif // WEBRTC_VIDEO_DECODER_H_	111	#endif // WEBRTC_VIDEO_DECODER_H_
76	-- a/third_party/webrtc/webrtc_tests.gypi	112	++ b/third_party/webrtc/webrtc_tests.gypi
Lines 148-153 Link Here
148	'video/bitrate_estimator_tests.cc',	148	'video/bitrate_estimator_tests.cc',
149	'video/end_to_end_tests.cc',	149	'video/end_to_end_tests.cc',
150	'video/send_statistics_proxy_unittest.cc',	150	'video/send_statistics_proxy_unittest.cc',
		151	'video/video_decoder_unittest.cc',
151	'video/video_send_stream_tests.cc',	152	'video/video_send_stream_tests.cc',
152	],	153	],
153	'dependencies': [	154	'dependencies': [