Home
last modified time | relevance | path

Searched refs:quantized (Results 1 – 25 of 46) sorted by relevance

12

/third_party/ffmpeg/libavcodec/
Daacenc_quantization.h91 float quantized, rd = 0.0f; in quantize_and_encode_band_cost_template() local
104 quantized = CLIPPED_ESCAPE; in quantize_and_encode_band_cost_template()
108 quantized = c*cbrtf(c)*IQ; in quantize_and_encode_band_cost_template()
112 quantized = vec[j]*IQ; in quantize_and_encode_band_cost_template()
114 di = t - quantized; in quantize_and_encode_band_cost_template()
116 out[i+j] = in[i+j] >= 0 ? quantized : -quantized; in quantize_and_encode_band_cost_template()
119 qenergy += quantized*quantized; in quantize_and_encode_band_cost_template()
124 quantized = vec[j]*IQ; in quantize_and_encode_band_cost_template()
125 qenergy += quantized*quantized; in quantize_and_encode_band_cost_template()
127 out[i+j] = quantized; in quantize_and_encode_band_cost_template()
[all …]
Ddcaenc.c87 int32_t quantized[MAX_CHANNELS][DCAENC_SUBBANDS][SUBBAND_SAMPLES]; member
658 c->adpcm_history[ch][band] + 4, c->quantized[ch][band], in quantize_adpcm_subband()
682 c->quantized[ch][band][sample] = val; in quantize_pcm()
689 static void accumulate_huff_bit_consumption(int abits, int32_t *quantized, in accumulate_huff_bit_consumption() argument
694 result[sel] += ff_dca_vlc_calc_quant_bits(quantized, SUBBAND_SAMPLES, in accumulate_huff_bit_consumption()
826 c->quantized[ch][band], in init_quantization_noise()
915 c->quantized[ch][band]+12, step_size, in fill_in_adpcm_bufer()
1080 ff_dca_vlc_enc_quant(&c->pb, &c->quantized[ch][band][ss * 8], 8, in put_subframe_samples()
1091 sum += c->quantized[ch][band][ss * 8 + i + j]; in put_subframe_samples()
1102 put_sbits(&c->pb, bits, c->quantized[ch][band][ss * 8 + i]); in put_subframe_samples()
/third_party/mindspore/mindspore/lite/tools/converter/quantizer/config/
Dfull_quant.cfg7 # Layers with size of weights exceeds threshold `min_quant_weight_size` will be quantized.
9 …Layers with channel size of weights exceeds threshold `min_quant_weight_channel` will be quantized.
39 # Activation quantized method supports MAX_MIN or KL or REMOVAL_OUTLIER
Dfixed_bit_weight_quant.cfg7 # Layers with size of weights exceeds threshold `min_quant_weight_size` will be quantized.
9 …Layers with channel size of weights exceeds threshold `min_quant_weight_channel` will be quantized.
Dmixed_bit_weight_quant.cfg7 # Layers with size of weights exceeds threshold `min_quant_weight_size` will be quantized.
9 …Layers with channel size of weights exceeds threshold `min_quant_weight_channel` will be quantized.
/third_party/mindspore/mindspore/lite/micro/coder/opcoders/nnacl/int8/
Dsigmoid_int8_coder.cc36 const int32_t quantized = std::round(sigmoid_value / output_scale) + output_zp; in CalculateTableList() local
37 auto out_value = static_cast<int8_t>(std::max(std::min(quantized, max_value), min_value)); in CalculateTableList()
/third_party/mindspore/mindspore/lite/src/runtime/kernel/arm/int8/
Dsigmoid_int8.cc41 const int32_t quantized = std::round(sigmoid_value / output_scale) + output_zp; in CalculateTableList() local
42 int8_t out_value = static_cast<int8_t>(std::max(std::min(quantized, max_value), min_value)); in CalculateTableList()
/third_party/mindspore/mindspore/lite/test/config/
Dmodels_process_only.cfg25 #ml_table_detection.onnx: onnx quantized model
/third_party/openGLES/extensions/EXT/
DEXT_texture_filter_minmax.txt113 ... where frac(x) denotes the fractional part of <x> and may be quantized
125 where frac(x) denotes the fractional part of <x> and may be quantized to a
198 may be quantized to a fixed-point value with implementation-dependent
DEXT_depth_bounds_test.txt157 the comparison uses those quantized bounds.
/third_party/skia/third_party/externals/opengl-registry/extensions/EXT/
DEXT_texture_filter_minmax.txt113 ... where frac(x) denotes the fractional part of <x> and may be quantized
125 where frac(x) denotes the fractional part of <x> and may be quantized to a
198 may be quantized to a fixed-point value with implementation-dependent
/third_party/skia/third_party/externals/opengl-registry/extensions/AMD/
DAMD_sample_positions.txt118 The quantized sample positions may be queried by calling GetMultisamplefv.
211 RESOLVED: GetMultisamplefv will return the quantized sample positions,
/third_party/openGLES/extensions/AMD/
DAMD_sample_positions.txt118 The quantized sample positions may be queried by calling GetMultisamplefv.
211 RESOLVED: GetMultisamplefv will return the quantized sample positions,
/third_party/skia/third_party/externals/opengl-registry/extensions/ARB/
DARB_texture_filter_minmax.txt112 quantized to a fixed-point value with implementation-dependent
124 where frac(x) denotes the fractional part of <x> and may be quantized to
223 may be quantized to a fixed-point value with implementation-dependent
/third_party/openGLES/extensions/ARB/
DARB_texture_filter_minmax.txt122 quantized to a fixed-point value with implementation-dependent
134 where frac(x) denotes the fractional part of <x> and may be quantized to
233 may be quantized to a fixed-point value with implementation-dependent
/third_party/boost/libs/math/doc/background/
Derror.qbk26 All the test programs calculate /quantized relative error/, whereas the graphs
/third_party/skia/third_party/externals/opengl-registry/extensions/OES/
DOES_vertex_type_10_10_10_2.txt54 stream which can be used to store normals or other attributes in a quantized
/third_party/openGLES/extensions/OES/
DOES_vertex_type_10_10_10_2.txt64 stream which can be used to store normals or other attributes in a quantized
/third_party/mindspore/mindspore/lite/tools/converter/parser/tflite/
Dschema.fbs57 // Parameters for converting a quantized tensor back to float.
60 // Given a quantized value q, the corresponding float value f should be:
77 // will be quantized across the second dimension of t.
/third_party/skia/third_party/externals/spirv-tools/source/opt/
Dconst_folding_rules.cpp443 utils::HexFloat<utils::FloatProxy<utils::Float16>> quantized(0); in FoldQuantizeToF16Scalar() local
445 orignal.castTo(quantized, utils::round_direction::kToZero); in FoldQuantizeToF16Scalar()
446 quantized.castTo(result, utils::round_direction::kToZero); in FoldQuantizeToF16Scalar()
/third_party/mindspore/third_party/proto/tensorflow/lite/
Dschema.fbs57 // Parameters for converting a quantized tensor back to float.
60 // Given a quantized value q, the corresponding float value f should be:
77 // will be quantized across the second dimension of t.
Dschema_2.4.1.fbs60 // Parameters for converting a quantized tensor back to float.
63 // Given a quantized value q, the corresponding float value f should be:
80 // will be quantized across the second dimension of t.
/third_party/gstreamer/gstplugins_bad/gst/dvbsubenc/libimagequant/
DREADME.md222 Returns pointer to palette optimized for image that has been quantized or remapped (final refinemen…
459 It must be called before the image is quantized.
461 Returns error if more than 256 colors are added. If image is quantized to fewer colors than the num…
/third_party/skia/third_party/externals/swiftshader/third_party/SPIRV-Tools/source/opt/
Dconst_folding_rules.cpp499 utils::HexFloat<utils::FloatProxy<utils::Float16>> quantized(0); in FoldQuantizeToF16Scalar() local
501 orignal.castTo(quantized, utils::round_direction::kToZero); in FoldQuantizeToF16Scalar()
502 quantized.castTo(result, utils::round_direction::kToZero); in FoldQuantizeToF16Scalar()
/third_party/spirv-tools/source/opt/
Dconst_folding_rules.cpp443 utils::HexFloat<utils::FloatProxy<utils::Float16>> quantized(0); in FoldQuantizeToF16Scalar() local
445 orignal.castTo(quantized, utils::round_direction::kToZero); in FoldQuantizeToF16Scalar()
446 quantized.castTo(result, utils::round_direction::kToZero); in FoldQuantizeToF16Scalar()

12