/* * Copyright (c) 2023, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 3-Clause Clear License * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear * License was not distributed with this source code in the LICENSE file, you * can obtain it at www.aomedia.org/license/software-license/bsd-3-c-c. If the * Alliance for Open Media Patent License 1.0 was not distributed with this * source code in the PATENTS file, you can obtain it at * www.aomedia.org/license/patent. */ #include "iamf/cli/parameter_block_partitioner.h" #include #include #include #include #include "absl/log/log.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "iamf/cli/cli_util.h" #include "iamf/cli/proto/ia_sequence_header.pb.h" #include "iamf/cli/proto/obu_header.pb.h" #include "iamf/cli/proto/parameter_block.pb.h" #include "iamf/cli/proto/parameter_data.pb.h" #include "iamf/common/utils/macros.h" #include "iamf/common/utils/numeric_utils.h" #include "iamf/common/utils/obu_util.h" #include "iamf/obu/types.h" namespace iamf_tools { using iamf_tools_cli_proto::ParameterBlockObuMetadata; namespace { absl::Status InterpolateMixGainParameterData( const iamf_tools_cli_proto::MixGainParameterData& mix_gain_parameter_data, InternalTimestamp start_time, InternalTimestamp end_time, InternalTimestamp target_time, int16_t& target_mix_gain) { const auto& param_data = mix_gain_parameter_data.param_data(); float target_mix_gain_db = 0; RETURN_IF_NOT_OK(InterpolateMixGainValue( mix_gain_parameter_data.animation_type(), iamf_tools_cli_proto::ANIMATE_STEP, iamf_tools_cli_proto::ANIMATE_LINEAR, iamf_tools_cli_proto::ANIMATE_BEZIER, [¶m_data]() { return static_cast(param_data.step().start_point_value()); }, [¶m_data]() { return static_cast(param_data.linear().start_point_value()); }, [¶m_data]() { return static_cast(param_data.linear().end_point_value()); }, [¶m_data]() { return static_cast(param_data.bezier().start_point_value()); }, [¶m_data]() { return static_cast(param_data.bezier().end_point_value()); }, [¶m_data]() { return static_cast(param_data.bezier().control_point_value()); }, [¶m_data]() { return static_cast( param_data.bezier().control_point_relative_time()); }, start_time, end_time, target_time, target_mix_gain_db)); // Interpolated values are in dB in float; convert back to Q7.8 to store in // the partitioned parameter block. return FloatToQ7_8(target_mix_gain_db, target_mix_gain); } /*!\brief Partitions a `MixGainParameterData` * * Partitions a `MixGainParameterData` including the nested fields that describe * animation. * * \param subblock_mix_gain Input mix gain to partition. * \param subblock_start_time Start time of the subblock being partitioned. * \param subblock_end_time End time of the subblock being partitioned. * \param partition_start_time Start time of the output partitioned parameter * block. * \param partitioned_end_time End time of the output partitioned parameter * block. * \param partitioned_subblock Output argument with `MixGainParameterData` * filled in. * \return `absl::OkStatus()` on success. A specific status on failure. */ absl::Status PartitionMixGain( const iamf_tools_cli_proto::MixGainParameterData& subblock_mix_gain, InternalTimestamp subblock_start_time, InternalTimestamp subblock_end_time, InternalTimestamp partitioned_start_time, InternalTimestamp partitioned_end_time, iamf_tools_cli_proto::ParameterSubblock& partitioned_subblock) { // Copy over the animation type. auto* mix_gain_param_data = partitioned_subblock.mutable_mix_gain_parameter_data(); mix_gain_param_data->set_animation_type(subblock_mix_gain.animation_type()); // Partition the animated parameter. switch (subblock_mix_gain.animation_type()) { using enum iamf_tools_cli_proto::AnimationType; case ANIMATE_STEP: { int16_t start_point_value; RETURN_IF_NOT_OK(InterpolateMixGainParameterData( subblock_mix_gain, subblock_start_time, subblock_end_time, partitioned_start_time, start_point_value)); mix_gain_param_data->mutable_param_data() ->mutable_step() ->set_start_point_value(static_cast(start_point_value)); return absl::OkStatus(); } case ANIMATE_LINEAR: { // Set partitioned start time to the value of the parameter at that time. LOG_FIRST_N(INFO, 3) << subblock_start_time << " " << subblock_end_time << " " << partitioned_start_time << " " << partitioned_end_time; // Calculate the subblock's parameter value at the start of the partition. int16_t start_point_value; int16_t end_point_value; auto status = InterpolateMixGainParameterData( subblock_mix_gain, subblock_start_time, subblock_end_time, partitioned_start_time, start_point_value); // Calculate the subblock's parameter value at the end of the partition. status.Update(InterpolateMixGainParameterData( subblock_mix_gain, subblock_start_time, subblock_end_time, partitioned_end_time, end_point_value)); if (!status.ok()) { return absl::Status( status.code(), absl::StrCat("Failed to interpolate mix gain values. " "`InterpolateMixGainParameterData` returned: ", status.message())); } auto* linear = mix_gain_param_data->mutable_param_data()->mutable_linear(); linear->set_start_point_value(static_cast(start_point_value)); linear->set_end_point_value(static_cast(end_point_value)); return absl::OkStatus(); } case ANIMATE_BEZIER: { if (subblock_start_time == partitioned_start_time && subblock_end_time == partitioned_end_time) { // Handle the simplest case where the subblock is aligned and does not // need partitioning. *mix_gain_param_data->mutable_param_data() = subblock_mix_gain.param_data(); return absl::OkStatus(); } // TODO(b/279581032): Carefully split the bezier curve. Be careful with // Q7.8 format. return absl::InvalidArgumentError(absl::StrCat( "The encoder does not fully support partitioning bezier ", "parameters yet.")); } default: return absl::InvalidArgumentError( absl::StrCat("Unrecognized animation type = ", subblock_mix_gain.animation_type())); } } /*!\brief Gets the subblocks that overlap with the input times. * * Finds all subblocks in the input Parameter Block that overlap with * `partitioned_start_time` and `partitioned_end_time. * * \param full_parameter_block Input full parameter block. * \param partitioned_start_time Start time of the output partitioned parameter * block. * \param partitioned_end_time End time of the output partitioned parameter * block. * \param partitioned_subblocks Output partitioned subblocks. * \param constant_subblock_duration Output argument which corresponds to the * value of `constant_subblock_duration` in the OBU or metadata. * \return `absl::OkStatus()` on success. A specific status on failure. */ absl::Status GetPartitionedSubblocks( const ParameterBlockObuMetadata& full_parameter_block, InternalTimestamp partitioned_start_time, InternalTimestamp partitioned_end_time, std::list& partitioned_subblocks, uint32_t& constant_subblock_duration) { LOG_FIRST_N(INFO, 1) << " full_parameter_block=\n" << full_parameter_block; InternalTimestamp current_time = full_parameter_block.start_timestamp(); // Track that the split subblocks cover the whole partition. int32_t total_covered_duration = 0; // Loop through all subblocks in the original Parameter Block. const auto num_subblocks = full_parameter_block.num_subblocks(); for (int i = 0; i < num_subblocks; ++i) { // Get the start and end time of this subblock. const int32_t subblock_start_time = current_time; // The partitioner works directly on the parameter block OBU metadata and // assumes all needed information (e.g. subblock duration) is in the // metadata themselves and does not support getting the information from // parameter definitions (i.e. parameter definition mode == 0). constexpr uint8_t kParamDefinitionModeOne = 1; const auto subblock_duration = GetParameterSubblockDuration( i, full_parameter_block.num_subblocks(), full_parameter_block.constant_subblock_duration(), full_parameter_block.duration(), kParamDefinitionModeOne, [&full_parameter_block](int i) { return full_parameter_block.subblocks(i).subblock_duration(); }, [](int i) { return absl::InvalidArgumentError( "Parameter Block Partitioner does not support the case where " "`param_definition_mode == 0"); }); if (!subblock_duration.ok()) { return subblock_duration.status(); } const int32_t subblock_end_time = subblock_start_time + static_cast(*subblock_duration); current_time = subblock_end_time; if (subblock_end_time <= partitioned_start_time || partitioned_end_time <= subblock_start_time) { // The subblock ends before the partition or starts after. It can't // overlap. continue; } // Found an overlapping subblock. Create a new one for the partition that // represents the overlapped time. iamf_tools_cli_proto::ParameterSubblock partitioned_subblock; const int partitioned_subblock_start = std::max(partitioned_start_time, subblock_start_time); const int partitioned_subblock_end = std::min(partitioned_end_time, subblock_end_time); uint32_t partitioned_subblock_duration = partitioned_subblock_end - partitioned_subblock_start; total_covered_duration += partitioned_subblock_duration; partitioned_subblock.set_subblock_duration(partitioned_subblock_duration); const auto& subblock_i = full_parameter_block.subblocks(i); if (subblock_i.has_mix_gain_parameter_data()) { // Mix Gain animated parameters need to be partitioned. RETURN_IF_NOT_OK(PartitionMixGain( subblock_i.mix_gain_parameter_data(), subblock_start_time, subblock_end_time, partitioned_subblock_start, partitioned_subblock_end, partitioned_subblock)); } else if (subblock_i.has_demixing_info_parameter_data()) { if (!partitioned_subblocks.empty()) { return absl::InvalidArgumentError( "There should only be one subblock for demixing info."); } *partitioned_subblock.mutable_demixing_info_parameter_data() = subblock_i.demixing_info_parameter_data(); } else if (subblock_i.has_recon_gain_info_parameter_data()) { if (!partitioned_subblocks.empty()) { return absl::InvalidArgumentError( "There should only be one subblock for recon gain info."); } *partitioned_subblock.mutable_recon_gain_info_parameter_data() = subblock_i.recon_gain_info_parameter_data(); } else { return absl::InvalidArgumentError("Unknown subblock type."); } partitioned_subblocks.push_back(partitioned_subblock); if (subblock_end_time >= partitioned_end_time) { // Subblock overlap is over. No more to find for this partition. break; } } const auto status = CompareTimestamps( partitioned_end_time - partitioned_start_time, total_covered_duration); if (!status.ok()) { return absl::Status( status.code(), absl::StrCat( "Unable to find enough subblocks to totally cover the duration " "of the partitioned Parameter Block OBU. Possible gap in the " "sequence. `CompareTimestamps` returned: ", status.message())); } // Get the value of `constant_subblock_duration`. std::vector subblock_durations; subblock_durations.reserve(partitioned_subblocks.size()); for (const auto& subblock : partitioned_subblocks) { subblock_durations.push_back(subblock.subblock_duration()); } constant_subblock_duration = ParameterBlockPartitioner::FindConstantSubblockDuration( subblock_durations); return absl::OkStatus(); } } // namespace uint32_t ParameterBlockPartitioner::FindConstantSubblockDuration( const std::vector& subblock_durations) { if (subblock_durations.empty()) { return 0; } const auto constant_subblock_duration = subblock_durations[0]; for (int i = 0; i < subblock_durations.size(); ++i) { if (i == subblock_durations.size() - 1 && subblock_durations[i] <= constant_subblock_duration) { // The duration of the final subblock can be implicitly calculated. As // long as it has equal or smaller duration than // `constant_subblock_duration`. continue; } // Other than the special case all subblocks must have the same duration. if (subblock_durations[i] != constant_subblock_duration) { return 0; } } return constant_subblock_duration; } absl::Status ParameterBlockPartitioner::FindPartitionDuration( const iamf_tools_cli_proto::ProfileVersion primary_profile, const iamf_tools_cli_proto::CodecConfigObuMetadata& codec_config_obu_metadata, uint32_t& partition_duration) { using enum iamf_tools_cli_proto::ProfileVersion; if (primary_profile != PROFILE_VERSION_SIMPLE && primary_profile != PROFILE_VERSION_BASE && primary_profile != PROFILE_VERSION_BASE_ENHANCED) { // This function only implements limitations described in IAMF v1.1.0 for // simple, base, base-enhanced profile. return absl::InvalidArgumentError( absl::StrCat("FindPartitionDuration() only works with Simple, Base, or " "Base-Enhanced profile")); } // TODO(b/283281856): Set the duration to a different value when // `parameter_rate != sample rate`. partition_duration = codec_config_obu_metadata.codec_config().num_samples_per_frame(); return absl::OkStatus(); } absl::Status ParameterBlockPartitioner::PartitionParameterBlock( const ParameterBlockObuMetadata& full_parameter_block, int32_t partitioned_start_time, int32_t partitioned_end_time, ParameterBlockObuMetadata& partitioned) { if (partitioned_start_time >= partitioned_end_time) { return absl::InvalidArgumentError( absl::StrCat("Cannot partition a parameter block with < 1 duration. " "(partitioned_start_time= ", partitioned_start_time, " partitioned_end_time=", partitioned_end_time)); } // Find the subblocks that overlap this partition. std::list partitioned_subblocks; uint32_t constant_subblock_duration; RETURN_IF_NOT_OK(GetPartitionedSubblocks( full_parameter_block, partitioned_start_time, partitioned_end_time, partitioned_subblocks, constant_subblock_duration)); // Create the partitioned parameter block OBU metadata. The first few fields // are always the same as the full metadata. partitioned.set_parameter_id(full_parameter_block.parameter_id()); partitioned.set_duration( static_cast(partitioned_end_time - partitioned_start_time)); partitioned.set_num_subblocks(partitioned_subblocks.size()); partitioned.set_constant_subblock_duration(constant_subblock_duration); *partitioned.mutable_obu_header() = full_parameter_block.obu_header(); for (const auto& subblock : partitioned_subblocks) { *partitioned.add_subblocks() = subblock; } partitioned.set_start_timestamp(partitioned_start_time); return absl::OkStatus(); } absl::Status ParameterBlockPartitioner::PartitionFrameAligned( uint32_t partition_duration, const ParameterBlockObuMetadata& full_parameter_block, std::list& partitioned_parameter_blocks) { // Partition this parameter block into several blocks with the same duration. const int32_t end_timestamp = full_parameter_block.start_timestamp() + full_parameter_block.duration(); for (int32_t t = full_parameter_block.start_timestamp(); t < end_timestamp; t += partition_duration) { LOG(INFO) << "Partitioning parameter blocks at timestamp= " << t; ParameterBlockObuMetadata partitioned_parameter_block; RETURN_IF_NOT_OK(PartitionParameterBlock(full_parameter_block, t, t + partition_duration, partitioned_parameter_block)); partitioned_parameter_blocks.push_back(partitioned_parameter_block); } return absl::OkStatus(); } } // namespace iamf_tools