1 /* 2 * Copyright (c) 2024 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef POST_PROCESSING_H 17 #define POST_PROCESSING_H 18 19 #include <array> 20 #include <memory> 21 #include <string> 22 #include <tuple> 23 #include <type_traits> 24 #include "surface.h" 25 #include "avcodec_errors.h" 26 #include "codecbase.h" 27 #include "meta/format.h" 28 #include "media_description.h" 29 #include "controller.h" 30 #include "dynamic_controller.h" 31 #include "state_machine.h" 32 #include "post_processing_utils.h" 33 #include "post_processing_callback.h" 34 #include "native_buffer.h" 35 36 namespace OHOS { 37 namespace MediaAVCodec { 38 namespace PostProcessing { 39 40 template<typename T, typename = IsDerivedController<T>> 41 class PostProcessing { 42 public: Create(const std::shared_ptr<CodecBase> codec,const Format & format,int32_t & ret)43 static std::unique_ptr<PostProcessing<T>> Create(const std::shared_ptr<CodecBase> codec, 44 const Format& format, int32_t& ret) 45 { 46 auto p = std::make_unique<PostProcessing<T>>(codec); 47 if (!p) { 48 AVCODEC_LOGE("Create post processing failed"); 49 ret = AVCS_ERR_NO_MEMORY; 50 return nullptr; 51 } 52 ret = p->Init(format); 53 if (ret != AVCS_ERR_OK) { 54 return nullptr; 55 } 56 return p; 57 } 58 PostProcessing(std::shared_ptr<CodecBase> codec)59 explicit PostProcessing(std::shared_ptr<CodecBase> codec) : codec_(codec) {} 60 ~PostProcessing()61 ~PostProcessing() 62 { 63 callbackUserData_ = nullptr; 64 } 65 SetCallback(const Callback & callback,void * userData)66 int32_t SetCallback(const Callback& callback, void* userData) 67 { 68 callback_ = callback; 69 callbackUserData_ = userData; 70 return AVCS_ERR_OK; 71 } 72 SetOutputSurface(sptr<Surface> surface)73 int32_t SetOutputSurface(sptr<Surface> surface) 74 { 75 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 76 switch (state_.Get()) { 77 case State::CONFIGURED: 78 { 79 config_.outputSurface = surface; 80 return AVCS_ERR_OK; 81 } 82 case State::PREPARED: 83 [[fallthrough]]; 84 case State::RUNNING: 85 [[fallthrough]]; 86 case State::FLUSHED: 87 [[fallthrough]]; 88 case State::STOPPED: 89 { 90 int32_t ret = controller_->SetOutputSurface(surface); 91 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Set output surface failed"); 92 config_.outputSurface = surface; 93 return ret; 94 } 95 default: 96 { 97 AVCODEC_LOGE("Invalid post processing state: %{public}s", state_.Name()); 98 return AVCS_ERR_UNKNOWN; 99 } 100 } 101 } 102 Prepare()103 int32_t Prepare() 104 { 105 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 106 CHECK_AND_RETURN_RET_LOG(state_.Get() == State::CONFIGURED, AVCS_ERR_INVALID_STATE, 107 "Invalid post processing state: %{public}s", state_.Name()); 108 CHECK_AND_RETURN_RET_LOG(config_.outputSurface != nullptr, AVCS_ERR_INVALID_OPERATION, 109 "Output surface is not set"); 110 111 int32_t ret = controller_->Create(); 112 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed"); 113 114 SetOutputSurfaceTransform(); 115 ret = controller_->SetOutputSurface(config_.outputSurface); 116 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed"); 117 118 ret = SetDecoderInputSurface(); 119 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed"); 120 121 ret = controller_->SetCallback(static_cast<void*>(&callback_), callbackUserData_); 122 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed"); 123 124 ret = ConfigureController(); 125 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed"); 126 127 ret = controller_->Prepare(); 128 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Prepare failed"); 129 130 state_.Set(State::PREPARED); 131 return AVCS_ERR_OK; 132 } 133 Start()134 int32_t Start() 135 { 136 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 137 CHECK_AND_RETURN_RET_LOG(state_.Get() == State::PREPARED || state_.Get() == State::STOPPED || 138 state_.Get() == State::FLUSHED, 139 AVCS_ERR_INVALID_OPERATION, "Post processing is not prepared"); 140 int32_t ret = controller_->Start(); 141 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Start failed"); 142 state_.Set(State::RUNNING); 143 return AVCS_ERR_OK; 144 } 145 Stop()146 int32_t Stop() 147 { 148 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 149 CHECK_AND_RETURN_RET_LOG(state_.Get() == State::RUNNING || state_.Get() == State::FLUSHED, 150 AVCS_ERR_INVALID_STATE, "Invalid post processing state: %{public}s", state_.Name()); 151 int32_t ret = controller_->Stop(); 152 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Start failed"); 153 state_.Set(State::STOPPED); 154 return AVCS_ERR_OK; 155 } 156 Flush()157 int32_t Flush() 158 { 159 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 160 CHECK_AND_RETURN_RET_LOG(state_.Get() == State::RUNNING, AVCS_ERR_INVALID_STATE, 161 "Invalid post processing state: %{public}s", state_.Name()); 162 int32_t ret = controller_->Flush(); 163 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Flush failed"); 164 state_.Set(State::FLUSHED); 165 return AVCS_ERR_OK; 166 } 167 Reset()168 int32_t Reset() 169 { 170 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 171 int32_t ret = controller_->Reset(); 172 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Reset failed"); 173 codec_.reset(); 174 state_.Set(State::DISABLED); 175 return AVCS_ERR_OK; 176 } 177 Release()178 int32_t Release() 179 { 180 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 181 config_.inputSurface = nullptr; 182 config_.outputSurface = nullptr; 183 controller_->Release(); 184 controller_->Destroy(); 185 controller_->UnloadInterfaces(); 186 codec_.reset(); 187 state_.Set(State::DISABLED); 188 return AVCS_ERR_OK; 189 } 190 ReleaseOutputBuffer(uint32_t index,bool render)191 int32_t ReleaseOutputBuffer(uint32_t index, bool render) 192 { 193 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 194 CHECK_AND_RETURN_RET_LOG(state_.Get() == State::RUNNING, AVCS_ERR_INVALID_STATE, 195 "Invalid post processing state: %{public}s", state_.Name()); 196 197 int32_t ret = controller_->ReleaseOutputBuffer(index, render); 198 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "ReleaseOutputBuffer failed"); 199 return AVCS_ERR_OK; 200 } 201 GetOutputFormat(Format & format)202 void GetOutputFormat(Format& format) 203 { 204 CHECK_AND_RETURN_LOG(controller_, "Post processing controller is null"); 205 206 int32_t ret = controller_->GetOutputFormat(format); 207 CHECK_AND_RETURN_LOG(ret == AVCS_ERR_OK, "GetOutputFormat failed"); 208 return; 209 } 210 NotifyEos()211 int32_t NotifyEos() 212 { 213 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_UNKNOWN, "Post processing controller is null"); 214 CHECK_AND_RETURN_RET_LOG(state_.Get() == State::RUNNING, AVCS_ERR_INVALID_STATE, 215 "Invalid post processing state: %{public}s", state_.Name()); 216 217 int32_t ret = controller_->NotifyEos(); 218 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "NotifyEos failed"); 219 return AVCS_ERR_OK; 220 } 221 private: Init(const Format & format)222 int32_t Init(const Format& format) 223 { 224 controller_ = std::make_unique<T>(); 225 CHECK_AND_RETURN_RET_LOG(controller_, AVCS_ERR_NO_MEMORY, "Create post processing controller failed"); 226 CHECK_AND_RETURN_RET_LOG(controller_->LoadInterfaces(), AVCS_ERR_VIDEO_UNSUPPORT_COLOR_SPACE_CONVERSION, 227 "Initialize interfaces failed."); 228 229 CreateConfiguration(format); 230 231 CapabilityInfo output{ 232 .colorSpaceType = config_.outputColorSpaceType, 233 .metadataType = config_.outputMetadataType, 234 .pixelFormat = config_.outputPixelFormat 235 }; 236 237 constexpr int32_t hdrVividVideoColorSpaceTypeList[]{ 238 0x440504, // BT2020 HLG Limit 239 0x440404, // BT2020 PQ Limit 240 0x240504 // BT2020 HLG Full 241 }; 242 constexpr int32_t hdrVividVideoMetadataType{3}; // HDR Vivid Video 243 constexpr int32_t hdrVividVideoPixelFormatList[]{35, 36}; 244 245 CapabilityInfo input; 246 input.metadataType = hdrVividVideoMetadataType; 247 bool supported{false}; 248 for (auto colorSpaceType : hdrVividVideoColorSpaceTypeList) { 249 for (auto pixelFormat : hdrVividVideoPixelFormatList) { 250 input.colorSpaceType = colorSpaceType, 251 input.pixelFormat = pixelFormat, 252 supported |= controller_->IsColorSpaceConversionSupported(input, output); 253 } 254 } 255 CHECK_AND_RETURN_RET_LOG(supported, AVCS_ERR_UNSUPPORT, "No capability found"); 256 257 state_.Set(State::CONFIGURED); 258 return AVCS_ERR_OK; 259 } 260 CreateConfiguration(const Format & format)261 void CreateConfiguration(const Format& format) 262 { 263 format_ = format; 264 int32_t width; 265 (void)format.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width); 266 int32_t height; 267 (void)format.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height); 268 int32_t colorSpaceType; 269 if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_VIDEO_DECODER_OUTPUT_COLOR_SPACE, colorSpaceType)) { 270 colorSpaceType = OH_NativeBuffer_ColorSpace::OH_COLORSPACE_BT709_LIMIT; // default to BT709 Limited 271 (void)format_.PutIntValue(MediaDescriptionKey::MD_KEY_VIDEO_DECODER_OUTPUT_COLOR_SPACE, colorSpaceType); 272 } 273 int32_t pixelFormat; 274 if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, pixelFormat)) { 275 pixelFormat = static_cast<int32_t>(VideoPixelFormat::NV12); 276 (void)format_.PutIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, pixelFormat); 277 } 278 int32_t rotation; 279 if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_ROTATION_ANGLE, rotation)) { 280 rotation = 0; // rotation 0 281 } 282 int32_t scalingMode; 283 if (!format.GetIntValue(MediaDescriptionKey::MD_KEY_SCALE_TYPE, scalingMode)) { 284 scalingMode = 0; // No scaling 285 } 286 switch (colorSpaceType) { 287 case static_cast<int32_t>(OH_NativeBuffer_ColorSpace::OH_COLORSPACE_BT709_LIMIT): 288 config_.outputColorSpaceType = ConfigurationParameters::colorSpaceBT709Limited; 289 break; 290 case static_cast<int32_t>(OH_NativeBuffer_ColorSpace::OH_COLORSPACE_P3_FULL): 291 config_.outputColorSpaceType = ConfigurationParameters::colorSpaceP3Full; 292 break; 293 default: 294 AVCODEC_LOGE("Unsupported color space type %{public}d", colorSpaceType); 295 } 296 switch (pixelFormat) { 297 case static_cast<int32_t>(VideoPixelFormat::NV12): 298 config_.outputPixelFormat = ConfigurationParameters::pixelFormatNV12; 299 break; 300 case static_cast<int32_t>(VideoPixelFormat::NV21): 301 config_.outputPixelFormat = ConfigurationParameters::pixelFormatNV21; 302 break; 303 default: 304 AVCODEC_LOGE("Unsupported pixel format %{public}d", pixelFormat); 305 } 306 config_.width = width; 307 config_.height = height; 308 config_.outputMetadataType = 0; // see OH_COLORSPACE_NONE 309 config_.rotation = rotation; 310 config_.scalingMode = scalingMode; 311 } 312 SetDecoderInputSurface()313 int32_t SetDecoderInputSurface() 314 { 315 CHECK_AND_RETURN_RET_LOG(codec_ != nullptr, AVCS_ERR_UNKNOWN, "Decoder is not found"); 316 sptr<Surface> surface = nullptr; 317 int32_t ret = controller_->CreateInputSurface(surface); 318 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK && surface != nullptr, ret, "Create input surface failed"); 319 GSError gsRet = surface->SetSurfaceSourceType(OHSurfaceSource::OH_SURFACE_SOURCE_VIDEO); 320 EXPECT_AND_LOGW(gsRet != GSERROR_OK, "Set surface source type failed, %{public}s", GSErrorStr(gsRet).c_str()); 321 ret = codec_->SetOutputSurface(surface); 322 CHECK_AND_RETURN_RET_LOG(ret == AVCS_ERR_OK, ret, "Set output surface of decoder failed"); 323 config_.inputSurface = surface; 324 return ret; 325 } 326 SetOutputSurfaceTransform()327 void SetOutputSurfaceTransform() 328 { 329 CHECK_AND_RETURN_LOG(config_.outputSurface != nullptr, "Output surface is null"); 330 331 GSError err{GSERROR_OK}; 332 switch (config_.rotation) { 333 case VIDEO_ROTATION_0: 334 err = config_.outputSurface->SetTransform(GRAPHIC_ROTATE_NONE); 335 break; 336 case VIDEO_ROTATION_90: 337 err = config_.outputSurface->SetTransform(GRAPHIC_ROTATE_270); 338 break; 339 case VIDEO_ROTATION_180: 340 err = config_.outputSurface->SetTransform(GRAPHIC_ROTATE_180); 341 break; 342 case VIDEO_ROTATION_270: 343 err = config_.outputSurface->SetTransform(GRAPHIC_ROTATE_90); 344 break; 345 default: 346 break; 347 } 348 if (err != GSERROR_OK) { 349 AVCODEC_LOGE("Set transform failed"); 350 } 351 352 switch (config_.scalingMode) { 353 case SCALING_MODE_SCALE_TO_WINDOW: 354 [[fallthrough]]; 355 case SCALING_MODE_SCALE_CROP: 356 err = config_.outputSurface->SetScalingMode(static_cast<ScalingMode>(config_.scalingMode)); 357 if (err != GSERROR_OK) { 358 AVCODEC_LOGE("Set transform failed"); 359 } 360 break; 361 default: 362 break; 363 } 364 } 365 366 class ConfigurationParameters { 367 public: 368 // PixelFormat 369 static constexpr int32_t pixelFormatNV12{24}; // NATIVEBUFFER_PIXEL_FMT_YCBCR_420_SP 370 static constexpr int32_t pixelFormatNV21{25}; // NATIVEBUFFER_PIXEL_FMT_YCRCB_420_SP 371 // ColorSpaceType 372 static constexpr int32_t colorSpaceBT709Limited{0x410101}; // OH_COLORSPACE_BT709_LIMIT 373 static constexpr int32_t colorSpaceP3Full{0x230206}; // OH_COLORSPACE_P3_FULL 374 // Primaries 375 static constexpr int32_t primariesBT709Limited{1}; 376 static constexpr int32_t primariesP3Full{6}; 377 // TransferFunction 378 static constexpr int32_t transFuncBT709Limited{1}; 379 static constexpr int32_t transFuncP3Full{2}; 380 // Matrix 381 static constexpr int32_t matrixBT709Limited{1}; 382 static constexpr int32_t matrixP3Full{3}; 383 // Range 384 static constexpr int32_t rangeBT709Limited{2}; 385 static constexpr int32_t rangeP3Full{1}; 386 // MetadataType 387 static constexpr int32_t metadataType{0}; 388 // RenderIntent 389 static constexpr int32_t renderIntent{2}; 390 }; 391 ConfigureController()392 int32_t ConfigureController() 393 { 394 constexpr std::string_view keyPrimaries{"colorspace_primaries"}; 395 constexpr std::string_view keyTransFunc{"colorspace_trans_func"}; 396 constexpr std::string_view keyMatrix{"colorspace_matrix"}; 397 constexpr std::string_view keyRange{"colorspace_range"}; 398 constexpr std::string_view keyMetadataType{"hdr_metadata_type"}; 399 constexpr std::string_view keyRenderIntent{"render_intent"}; 400 constexpr std::string_view keyPixelFormat{"pixel_format"}; 401 402 Format format(format_); 403 if (config_.outputColorSpaceType == ConfigurationParameters::colorSpaceBT709Limited) { 404 format.PutIntValue(keyPrimaries, ConfigurationParameters::primariesBT709Limited); 405 format.PutIntValue(keyTransFunc, ConfigurationParameters::transFuncBT709Limited); 406 format.PutIntValue(keyMatrix, ConfigurationParameters::matrixBT709Limited); 407 format.PutIntValue(keyRange, ConfigurationParameters::rangeBT709Limited); 408 } else if (config_.outputColorSpaceType == ConfigurationParameters::colorSpaceP3Full) { 409 format.PutIntValue(keyPrimaries, ConfigurationParameters::primariesP3Full); 410 format.PutIntValue(keyTransFunc, ConfigurationParameters::transFuncP3Full); 411 format.PutIntValue(keyMatrix, ConfigurationParameters::matrixP3Full); 412 format.PutIntValue(keyRange, ConfigurationParameters::rangeP3Full); 413 } 414 format.PutIntValue(keyMetadataType, ConfigurationParameters::metadataType); 415 format.PutIntValue(keyRenderIntent, ConfigurationParameters::renderIntent); 416 format.PutIntValue(keyPixelFormat, config_.outputPixelFormat); 417 418 return controller_->Configure(format); 419 } 420 421 struct Configuration { 422 int32_t width{0}; 423 int32_t height{0}; 424 int32_t inputColorSpaceType{0}; 425 int32_t inputMetadataType{0}; 426 int32_t inputPixelFormat{0}; 427 sptr<Surface> inputSurface{nullptr}; 428 int32_t outputColorSpaceType{0}; 429 int32_t outputMetadataType{0}; 430 int32_t outputPixelFormat{0}; 431 sptr<Surface> outputSurface{nullptr}; 432 int32_t rotation{0}; 433 int32_t scalingMode{0}; 434 }; 435 436 StateMachine state_; 437 Configuration config_; 438 Format format_; 439 std::unique_ptr<Controller<T>> controller_{nullptr}; 440 std::shared_ptr<CodecBase> codec_{nullptr}; 441 static constexpr HiviewDFX::HiLogLabel LABEL{LogLabel("PostProcessing")}; 442 Callback callback_; 443 void* callbackUserData_{nullptr}; 444 }; 445 446 using DynamicPostProcessing = PostProcessing<DynamicController>; 447 448 } // namespace PostProcessing 449 } // namespace OHOS 450 } // namespace MediaAVCodec 451 452 #endif // POST_PROCESSING_H