1 /* 2 * Copyright 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_C2_SOFT_VPX_ENC_H__ 18 #define ANDROID_C2_SOFT_VPX_ENC_H__ 19 20 #include <media/stagefright/foundation/MediaDefs.h> 21 22 #include <C2PlatformSupport.h> 23 #include <Codec2BufferUtils.h> 24 #include <SimpleC2Component.h> 25 #include <SimpleC2Interface.h> 26 #include <util/C2InterfaceHelper.h> 27 28 #include "vpx/vpx_encoder.h" 29 #include "vpx/vpx_codec.h" 30 #include "vpx/vpx_image.h" 31 #include "vpx/vp8cx.h" 32 33 namespace android { 34 35 // TODO: These defs taken from deprecated OMX_VideoExt.h. Move these definitions 36 // to a new header file and include it. 37 38 /** Maximum number of temporal layers */ 39 #define MAXTEMPORALLAYERS 3 40 41 /** temporal layer patterns */ 42 typedef enum TemporalPatternType { 43 VPXTemporalLayerPatternNone = 0, 44 VPXTemporalLayerPatternWebRTC = 1, 45 VPXTemporalLayerPatternMax = 0x7FFFFFFF 46 } TemporalPatternType; 47 48 // Base class for a VPX Encoder Component 49 // 50 // Only following encoder settings are available (codec specific settings might 51 // be available in the sub-classes): 52 // - video resolution 53 // - target bitrate 54 // - rate control (constant / variable) 55 // - frame rate 56 // - error resilience 57 // - reconstruction & loop filters (g_profile) 58 // 59 // Only following color formats are recognized 60 // - C2PlanarLayout::TYPE_RGB 61 // - C2PlanarLayout::TYPE_RGBA 62 // 63 // Following settings are not configurable by the client 64 // - encoding deadline is realtime 65 // - multithreaded encoding utilizes a number of threads equal 66 // to online cpu's available 67 // - the algorithm interface for encoder is decided by the sub-class in use 68 // - fractional bits of frame rate is discarded 69 // - timestamps are in microseconds, therefore encoder timebase is fixed 70 // to 1/1000000 71 72 struct C2SoftVpxEnc : public SimpleC2Component { 73 class IntfImpl; 74 75 C2SoftVpxEnc(const char* name, c2_node_id_t id, 76 const std::shared_ptr<IntfImpl>& intfImpl); 77 78 // From SimpleC2Component 79 c2_status_t onInit() override final; 80 c2_status_t onStop() override final; 81 void onReset() override final; 82 void onRelease() override final; 83 c2_status_t onFlush_sm() override final; 84 85 void process( 86 const std::unique_ptr<C2Work> &work, 87 const std::shared_ptr<C2BlockPool> &pool) override final; 88 c2_status_t drain( 89 uint32_t drainMode, 90 const std::shared_ptr<C2BlockPool> &pool) override final; 91 92 protected: 93 std::shared_ptr<IntfImpl> mIntf; 94 virtual ~C2SoftVpxEnc(); 95 96 // Initializes vpx encoder with available settings. 97 status_t initEncoder(); 98 99 // Populates mCodecInterface with codec specific settings. 100 virtual void setCodecSpecificInterface() = 0; 101 102 // Sets codec specific configuration. 103 virtual void setCodecSpecificConfiguration() = 0; 104 105 // Sets codec specific encoder controls. 106 virtual vpx_codec_err_t setCodecSpecificControls() = 0; 107 108 // Get current encode flags. 109 virtual vpx_enc_frame_flags_t getEncodeFlags(); 110 111 enum TemporalReferences { 112 // For 1 layer case: reference all (last, golden, and alt ref), but only 113 // update last. 114 kTemporalUpdateLastRefAll = 12, 115 // First base layer frame for 3 temporal layers, which updates last and 116 // golden with alt ref dependency. 117 kTemporalUpdateLastAndGoldenRefAltRef = 11, 118 // First enhancement layer with alt ref dependency. 119 kTemporalUpdateGoldenRefAltRef = 10, 120 // First enhancement layer with alt ref dependency. 121 kTemporalUpdateGoldenWithoutDependencyRefAltRef = 9, 122 // Base layer with alt ref dependency. 123 kTemporalUpdateLastRefAltRef = 8, 124 // Highest enhacement layer without dependency on golden with alt ref 125 // dependency. 126 kTemporalUpdateNoneNoRefGoldenRefAltRef = 7, 127 // Second layer and last frame in cycle, for 2 layers. 128 kTemporalUpdateNoneNoRefAltref = 6, 129 // Highest enhancement layer. 130 kTemporalUpdateNone = 5, 131 // Second enhancement layer. 132 kTemporalUpdateAltref = 4, 133 // Second enhancement layer without dependency on previous frames in 134 // the second enhancement layer. 135 kTemporalUpdateAltrefWithoutDependency = 3, 136 // First enhancement layer. 137 kTemporalUpdateGolden = 2, 138 // First enhancement layer without dependency on previous frames in 139 // the first enhancement layer. 140 kTemporalUpdateGoldenWithoutDependency = 1, 141 // Base layer. 142 kTemporalUpdateLast = 0, 143 }; 144 enum { 145 kMaxTemporalPattern = 8 146 }; 147 148 // vpx specific opaque data structure that 149 // stores encoder state 150 vpx_codec_ctx_t* mCodecContext; 151 152 // vpx specific data structure that 153 // stores encoder configuration 154 vpx_codec_enc_cfg_t* mCodecConfiguration; 155 156 // vpx specific read-only data structure 157 // that specifies algorithm interface (e.g. vp8) 158 vpx_codec_iface_t* mCodecInterface; 159 160 // align stride to the power of 2 161 int32_t mStrideAlign; 162 163 // Color format for the input port 164 vpx_img_fmt_t mColorFormat; 165 166 // Bitrate control mode, either constant or variable 167 vpx_rc_mode mBitrateControlMode; 168 169 // Parameter that denotes whether error resilience 170 // is enabled in encoder 171 bool mErrorResilience; 172 173 // Minimum (best quality) quantizer 174 uint32_t mMinQuantizer; 175 176 // Maximum (worst quality) quantizer 177 uint32_t mMaxQuantizer; 178 179 // Number of coding temporal layers to be used. 180 size_t mTemporalLayers; 181 182 // Temporal layer bitrare ratio in percentage 183 uint32_t mTemporalLayerBitrateRatio[MAXTEMPORALLAYERS]; 184 185 // Temporal pattern type 186 TemporalPatternType mTemporalPatternType; 187 188 // Temporal pattern length 189 size_t mTemporalPatternLength; 190 191 // Temporal pattern current index 192 size_t mTemporalPatternIdx; 193 194 // Frame type temporal pattern 195 TemporalReferences mTemporalPattern[kMaxTemporalPattern]; 196 197 // Last input buffer timestamp 198 uint64_t mLastTimestamp; 199 200 // Number of input frames 201 int64_t mNumInputFrames; 202 203 // Conversion buffer is needed to input to 204 // yuv420 planar format. 205 MemoryBlock mConversionBuffer; 206 207 // Signalled EOS 208 bool mSignalledOutputEos; 209 210 // Signalled Error 211 bool mSignalledError; 212 213 // configurations used by component in process 214 // (TODO: keep this in intf but make them internal only) 215 std::shared_ptr<C2StreamPictureSizeInfo::input> mSize; 216 std::shared_ptr<C2StreamIntraRefreshTuning::output> mIntraRefresh; 217 std::shared_ptr<C2StreamFrameRateInfo::output> mFrameRate; 218 std::shared_ptr<C2StreamBitrateInfo::output> mBitrate; 219 std::shared_ptr<C2StreamBitrateModeTuning::output> mBitrateMode; 220 std::shared_ptr<C2StreamRequestSyncFrameTuning::output> mRequestSync; 221 222 C2_DO_NOT_COPY(C2SoftVpxEnc); 223 }; 224 225 class C2SoftVpxEnc::IntfImpl : public C2InterfaceHelper { 226 public: IntfImpl(const std::shared_ptr<C2ReflectorHelper> & helper)227 explicit IntfImpl(const std::shared_ptr<C2ReflectorHelper>& helper) 228 : C2InterfaceHelper(helper) { 229 setDerivedInstance(this); 230 231 addParameter( 232 DefineParam(mInputFormat, C2_NAME_INPUT_STREAM_FORMAT_SETTING) 233 .withConstValue( 234 new C2StreamFormatConfig::input(0u, C2FormatVideo)) 235 .build()); 236 237 addParameter( 238 DefineParam(mOutputFormat, C2_NAME_OUTPUT_STREAM_FORMAT_SETTING) 239 .withConstValue( 240 new C2StreamFormatConfig::output(0u, C2FormatCompressed)) 241 .build()); 242 243 addParameter( 244 DefineParam(mInputMediaType, C2_NAME_INPUT_PORT_MIME_SETTING) 245 .withConstValue(AllocSharedString<C2PortMimeConfig::input>( 246 MEDIA_MIMETYPE_VIDEO_RAW)) 247 .build()); 248 249 addParameter( 250 DefineParam(mOutputMediaType, C2_NAME_OUTPUT_PORT_MIME_SETTING) 251 .withConstValue(AllocSharedString<C2PortMimeConfig::output>( 252 #ifdef VP9 253 MEDIA_MIMETYPE_VIDEO_VP9 254 #else 255 MEDIA_MIMETYPE_VIDEO_VP8 256 #endif 257 )) 258 .build()); 259 260 addParameter(DefineParam(mUsage, C2_NAME_INPUT_STREAM_USAGE_SETTING) 261 .withConstValue(new C2StreamUsageTuning::input( 262 0u, (uint64_t)C2MemoryUsage::CPU_READ)) 263 .build()); 264 265 addParameter( 266 DefineParam(mSize, C2_NAME_STREAM_VIDEO_SIZE_SETTING) 267 .withDefault(new C2VideoSizeStreamTuning::input(0u, 320, 240)) 268 .withFields({ 269 C2F(mSize, width).inRange(2, 2048, 2), 270 C2F(mSize, height).inRange(2, 2048, 2), 271 }) 272 .withSetter(SizeSetter) 273 .build()); 274 275 addParameter( 276 DefineParam(mBitrateMode, C2_PARAMKEY_BITRATE_MODE) 277 .withDefault(new C2StreamBitrateModeTuning::output( 278 0u, C2Config::BITRATE_CONST)) 279 .withFields({ 280 C2F(mBitrateMode, value).oneOf({ 281 C2Config::BITRATE_CONST, C2Config::BITRATE_VARIABLE }) 282 }) 283 .withSetter( 284 Setter<decltype(*mBitrateMode)>::StrictValueWithNoDeps) 285 .build()); 286 287 addParameter( 288 DefineParam(mFrameRate, C2_NAME_STREAM_FRAME_RATE_SETTING) 289 .withDefault(new C2StreamFrameRateInfo::output(0u, 30.)) 290 // TODO: More restriction? 291 .withFields({C2F(mFrameRate, value).greaterThan(0.)}) 292 .withSetter( 293 Setter<decltype(*mFrameRate)>::StrictValueWithNoDeps) 294 .build()); 295 296 addParameter( 297 DefineParam(mLayering, C2_PARAMKEY_TEMPORAL_LAYERING) 298 .withDefault(C2StreamTemporalLayeringTuning::output::AllocShared(0u, 0, 0, 0)) 299 .withFields({ 300 C2F(mLayering, m.layerCount).inRange(0, 4), 301 C2F(mLayering, m.bLayerCount).inRange(0, 0), 302 C2F(mLayering, m.bitrateRatios).inRange(0., 1.) 303 }) 304 .withSetter(LayeringSetter) 305 .build()); 306 307 addParameter( 308 DefineParam(mSyncFramePeriod, C2_PARAMKEY_SYNC_FRAME_INTERVAL) 309 .withDefault(new C2StreamSyncFrameIntervalTuning::output(0u, 1000000)) 310 .withFields({C2F(mSyncFramePeriod, value).any()}) 311 .withSetter(Setter<decltype(*mSyncFramePeriod)>::StrictValueWithNoDeps) 312 .build()); 313 314 addParameter( 315 DefineParam(mBitrate, C2_NAME_STREAM_BITRATE_SETTING) 316 .withDefault(new C2BitrateTuning::output(0u, 64000)) 317 .withFields({C2F(mBitrate, value).inRange(4096, 40000000)}) 318 .withSetter(BitrateSetter) 319 .build()); 320 321 addParameter( 322 DefineParam(mIntraRefresh, C2_PARAMKEY_INTRA_REFRESH) 323 .withConstValue(new C2StreamIntraRefreshTuning::output( 324 0u, C2Config::INTRA_REFRESH_DISABLED, 0.)) 325 .build()); 326 327 addParameter( 328 DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) 329 .withDefault(new C2StreamProfileLevelInfo::output( 330 0u, PROFILE_VP9_0, LEVEL_VP9_4_1)) 331 .withFields({ 332 C2F(mProfileLevel, profile).equalTo( 333 PROFILE_VP9_0 334 ), 335 C2F(mProfileLevel, level).equalTo( 336 LEVEL_VP9_4_1), 337 }) 338 .withSetter(ProfileLevelSetter) 339 .build()); 340 341 addParameter( 342 DefineParam(mRequestSync, C2_PARAMKEY_REQUEST_SYNC_FRAME) 343 .withDefault(new C2StreamRequestSyncFrameTuning::output(0u, C2_FALSE)) 344 .withFields({C2F(mRequestSync, value).oneOf({ C2_FALSE, C2_TRUE }) }) 345 .withSetter(Setter<decltype(*mRequestSync)>::NonStrictValueWithNoDeps) 346 .build()); 347 } 348 BitrateSetter(bool mayBlock,C2P<C2StreamBitrateInfo::output> & me)349 static C2R BitrateSetter(bool mayBlock, C2P<C2StreamBitrateInfo::output> &me) { 350 (void)mayBlock; 351 C2R res = C2R::Ok(); 352 if (me.v.value <= 4096) { 353 me.set().value = 4096; 354 } 355 return res; 356 } 357 SizeSetter(bool mayBlock,const C2P<C2StreamPictureSizeInfo::input> & oldMe,C2P<C2StreamPictureSizeInfo::input> & me)358 static C2R SizeSetter(bool mayBlock, const C2P<C2StreamPictureSizeInfo::input> &oldMe, 359 C2P<C2StreamPictureSizeInfo::input> &me) { 360 (void)mayBlock; 361 C2R res = C2R::Ok(); 362 if (!me.F(me.v.width).supportsAtAll(me.v.width)) { 363 res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.width))); 364 me.set().width = oldMe.v.width; 365 } 366 if (!me.F(me.v.height).supportsAtAll(me.v.height)) { 367 res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.height))); 368 me.set().height = oldMe.v.height; 369 } 370 return res; 371 } 372 ProfileLevelSetter(bool mayBlock,C2P<C2StreamProfileLevelInfo::output> & me)373 static C2R ProfileLevelSetter( 374 bool mayBlock, 375 C2P<C2StreamProfileLevelInfo::output> &me) { 376 (void)mayBlock; 377 if (!me.F(me.v.profile).supportsAtAll(me.v.profile)) { 378 me.set().profile = PROFILE_VP9_0; 379 } 380 if (!me.F(me.v.level).supportsAtAll(me.v.level)) { 381 me.set().level = LEVEL_VP9_4_1; 382 } 383 return C2R::Ok(); 384 } 385 LayeringSetter(bool mayBlock,C2P<C2StreamTemporalLayeringTuning::output> & me)386 static C2R LayeringSetter(bool mayBlock, C2P<C2StreamTemporalLayeringTuning::output>& me) { 387 (void)mayBlock; 388 C2R res = C2R::Ok(); 389 if (me.v.m.layerCount > 4) { 390 me.set().m.layerCount = 4; 391 } 392 me.set().m.bLayerCount = 0; 393 // ensure ratios are monotonic and clamped between 0 and 1 394 for (size_t ix = 0; ix < me.v.flexCount(); ++ix) { 395 me.set().m.bitrateRatios[ix] = c2_clamp( 396 ix > 0 ? me.v.m.bitrateRatios[ix - 1] : 0, me.v.m.bitrateRatios[ix], 1.); 397 } 398 ALOGI("setting temporal layering %u + %u", me.v.m.layerCount, me.v.m.bLayerCount); 399 return res; 400 } 401 402 // unsafe getters getSize_l()403 std::shared_ptr<C2StreamPictureSizeInfo::input> getSize_l() const { return mSize; } getIntraRefresh_l()404 std::shared_ptr<C2StreamIntraRefreshTuning::output> getIntraRefresh_l() const { return mIntraRefresh; } getFrameRate_l()405 std::shared_ptr<C2StreamFrameRateInfo::output> getFrameRate_l() const { return mFrameRate; } getBitrate_l()406 std::shared_ptr<C2StreamBitrateInfo::output> getBitrate_l() const { return mBitrate; } getBitrateMode_l()407 std::shared_ptr<C2StreamBitrateModeTuning::output> getBitrateMode_l() const { return mBitrateMode; } getRequestSync_l()408 std::shared_ptr<C2StreamRequestSyncFrameTuning::output> getRequestSync_l() const { return mRequestSync; } getTemporalLayers_l()409 std::shared_ptr<C2StreamTemporalLayeringTuning::output> getTemporalLayers_l() const { return mLayering; } getSyncFramePeriod()410 uint32_t getSyncFramePeriod() const { 411 if (mSyncFramePeriod->value < 0 || mSyncFramePeriod->value == INT64_MAX) { 412 return 0; 413 } 414 double period = mSyncFramePeriod->value / 1e6 * mFrameRate->value; 415 return (uint32_t)c2_max(c2_min(period + 0.5, double(UINT32_MAX)), 1.); 416 } 417 418 private: 419 std::shared_ptr<C2StreamFormatConfig::input> mInputFormat; 420 std::shared_ptr<C2StreamFormatConfig::output> mOutputFormat; 421 std::shared_ptr<C2PortMimeConfig::input> mInputMediaType; 422 std::shared_ptr<C2PortMimeConfig::output> mOutputMediaType; 423 std::shared_ptr<C2StreamUsageTuning::input> mUsage; 424 std::shared_ptr<C2VideoSizeStreamTuning::input> mSize; 425 std::shared_ptr<C2StreamFrameRateInfo::output> mFrameRate; 426 std::shared_ptr<C2StreamTemporalLayeringTuning::output> mLayering; 427 std::shared_ptr<C2StreamIntraRefreshTuning::output> mIntraRefresh; 428 std::shared_ptr<C2StreamRequestSyncFrameTuning::output> mRequestSync; 429 std::shared_ptr<C2StreamSyncFrameIntervalTuning::output> mSyncFramePeriod; 430 std::shared_ptr<C2BitrateTuning::output> mBitrate; 431 std::shared_ptr<C2StreamBitrateModeTuning::output> mBitrateMode; 432 std::shared_ptr<C2StreamProfileLevelInfo::output> mProfileLevel; 433 }; 434 435 } // namespace android 436 437 #endif // ANDROID_C2_SOFT_VPX_ENC_H__ 438