Searched refs:dequantized_value (Results 1 – 3 of 3) sorted by relevance
| /external/tensorflow/tensorflow/lite/delegates/hexagon/builders/ |
| D | conv_2d_helpers.cc | 108 const float dequantized_value = in ProcessPerChannelQuantizedBias() local
110 const float abs_dequantized_value = std::abs(dequantized_value); in ProcessPerChannelQuantizedBias()
114 dequantized_bias.push_back(dequantized_value); in ProcessPerChannelQuantizedBias()
|
| /external/tensorflow/tensorflow/lite/tools/optimize/ |
| D | quantize_model_test.cc | 666 auto dequantized_value = bias_values[i] * bias_scale; in TEST_F() local
667 EXPECT_NEAR(dequantized_value, bias_float_buffer[i], eps); in TEST_F()
683 auto dequantized_value = in TEST_F() local
685 EXPECT_NEAR(dequantized_value, weights_float_buffer[element_idx], eps); in TEST_F()
776 auto dequantized_value = bias_values[i] * bias_scales[i]; in TEST_P() local
777 EXPECT_NEAR(dequantized_value, bias_float_buffer[i], bias_scales[i] / 2); in TEST_P()
782 auto dequantized_value = bias_values[i] * bias_scales[i]; in TEST_P() local
783 EXPECT_NEAR(dequantized_value, bias_float_buffer[i], bias_scales[i] / 2); in TEST_P()
802 auto dequantized_value = weight_values[element_idx] * scale; in TEST_P() local
803 EXPECT_NEAR(dequantized_value, weights_float_buffer[element_idx], in TEST_P()
|
| /external/libgav1/libgav1/src/tile/ |
| D | tile.cc | 1337 int32_t dequantized_value = (static_cast<int64_t>(q) * level) & 0xffffff; in ReadSignAndApplyDequantization() local
1338 dequantized_value >>= shift; in ReadSignAndApplyDequantization()
1350 dequantized_value = std::min(dequantized_value - sign, max_value) ^ -sign; in ReadSignAndApplyDequantization()
1351 residual_buffer[pos] = dequantized_value; in ReadSignAndApplyDequantization()
|