| /external/python/cpython2/Lib/test/decimaltestdata/ |
| D | dqQuantize.decTest | 2 -- dqQuantize.decTest -- decQuad quantize operation --
24 -- 2004.03.15 Underflow for quantize is suppressed
26 -- [Forked from quantize.decTest 2006.11.25]
36 dqqua001 quantize 0 1e0 -> 0
37 dqqua002 quantize 1 1e0 -> 1
38 dqqua003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
39 dqqua005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
40 dqqua006 quantize 0.1 1e0 -> 0 Inexact Rounded
41 dqqua007 quantize 0.1 1e-1 -> 0.1
42 dqqua008 quantize 0.1 1e-2 -> 0.10
[all …]
|
| D | ddQuantize.decTest | 2 -- ddQuantize.decTest -- decDouble quantize operation --
24 -- 2004.03.15 Underflow for quantize is suppressed
34 ddqua001 quantize 0 1e0 -> 0
35 ddqua002 quantize 1 1e0 -> 1
36 ddqua003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
37 ddqua005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
38 ddqua006 quantize 0.1 1e0 -> 0 Inexact Rounded
39 ddqua007 quantize 0.1 1e-1 -> 0.1
40 ddqua008 quantize 0.1 1e-2 -> 0.10
41 ddqua009 quantize 0.1 1e-3 -> 0.100
[all …]
|
| D | quantize.decTest | 2 -- quantize.decTest -- decimal quantize operation --
24 -- 2004.03.15 Underflow for quantize is suppressed
34 quax001 quantize 0 1e0 -> 0
35 quax002 quantize 1 1e0 -> 1
36 quax003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
37 quax005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
38 quax006 quantize 0.1 1e0 -> 0 Inexact Rounded
39 quax007 quantize 0.1 1e-1 -> 0.1
40 quax008 quantize 0.1 1e-2 -> 0.10
41 quax009 quantize 0.1 1e-3 -> 0.100
[all …]
|
| D | ddCanonical.decTest | 287 ddcan401 quantize #6e38ff3ffff3fcff 1 -> #6e38ff3fcff3fcff
288 ddcan402 quantize #6e38ff3fcff3fdff 0 -> #6e38ff3fcff3fcff
289 ddcan403 quantize #7880000000000000 Inf -> #7800000000000000
290 ddcan404 quantize #7802000000000000 -Inf -> #7800000000000000
291 ddcan410 quantize #7c03ff3fcff3fcff 1 -> #7c00ff3fcff3fcff
292 ddcan411 quantize #7c03ff3fcff3fcff 1 -> #7c00ff3fcff3fcff
293 ddcan412 quantize #7c40ff3fcff3fcff 1 -> #7c00ff3fcff3fcff
294 ddcan413 quantize #7c40ff3fcff3fcff 1 -> #7c00ff3fcff3fcff
295 ddcan414 quantize #7e00ffffcff3fcff 1 -> #7c00ff3fcff3fcff Invalid_operation
296 ddcan415 quantize #7e00ffffcff3fcff 1 -> #7c00ff3fcff3fcff Invalid_operation
[all …]
|
| D | dqCanonical.decTest | 310 dqcan401 quantize #ee080ff3fcff3fcff3fffffffff3fcff 0 -> #ee080ff3fcff3fcff3fcff3fcff3fcff
311 dqcan402 quantize #ee080ff3fffffffffffcff3fcff3fcff 0 -> #ee080ff3fcff3fcff3fcff3fcff3fcff
312 dqcan403 quantize #78800000000000000000000000000000 Inf -> #78000000000000000000000000000000
313 dqcan404 quantize #78020000000000000000000000000000 -Inf -> #78000000000000000000000000000000
314 dqcan410 quantize #7c080ff3fcff3fcff3fcff3fcff3fcff 1 -> #7c000ff3fcff3fcff3fcff3fcff3fcff
315 dqcan411 quantize #fc000ff3fcfffffff3fcff3fcff3fcff 1 -> #fc000ff3fcff3fcff3fcff3fcff3fcff
316 dqcan412 quantize #7e100ff3fcff3fcff3fcff3fcff3fcff 1 -> #7c000ff3fcff3fcff3fcff3fcff3fcff Inva…
317 dqcan413 quantize #fe000ff3fcff3fcff3ffffffcff3fcff 1 -> #fc000ff3fcff3fcff3fcff3fcff3fcff Inva…
|
| /external/python/cpython3/Lib/test/decimaltestdata/ |
| D | dqQuantize.decTest | 2 -- dqQuantize.decTest -- decQuad quantize operation --
24 -- 2004.03.15 Underflow for quantize is suppressed
26 -- [Forked from quantize.decTest 2006.11.25]
36 dqqua001 quantize 0 1e0 -> 0
37 dqqua002 quantize 1 1e0 -> 1
38 dqqua003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
39 dqqua005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
40 dqqua006 quantize 0.1 1e0 -> 0 Inexact Rounded
41 dqqua007 quantize 0.1 1e-1 -> 0.1
42 dqqua008 quantize 0.1 1e-2 -> 0.10
[all …]
|
| D | ddQuantize.decTest | 2 -- ddQuantize.decTest -- decDouble quantize operation --
24 -- 2004.03.15 Underflow for quantize is suppressed
34 ddqua001 quantize 0 1e0 -> 0
35 ddqua002 quantize 1 1e0 -> 1
36 ddqua003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
37 ddqua005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
38 ddqua006 quantize 0.1 1e0 -> 0 Inexact Rounded
39 ddqua007 quantize 0.1 1e-1 -> 0.1
40 ddqua008 quantize 0.1 1e-2 -> 0.10
41 ddqua009 quantize 0.1 1e-3 -> 0.100
[all …]
|
| D | quantize.decTest | 2 -- quantize.decTest -- decimal quantize operation --
24 -- 2004.03.15 Underflow for quantize is suppressed
34 quax001 quantize 0 1e0 -> 0
35 quax002 quantize 1 1e0 -> 1
36 quax003 quantize 0.1 1e+2 -> 0E+2 Inexact Rounded
37 quax005 quantize 0.1 1e+1 -> 0E+1 Inexact Rounded
38 quax006 quantize 0.1 1e0 -> 0 Inexact Rounded
39 quax007 quantize 0.1 1e-1 -> 0.1
40 quax008 quantize 0.1 1e-2 -> 0.10
41 quax009 quantize 0.1 1e-3 -> 0.100
[all …]
|
| D | ddCanonical.decTest | 287 ddcan401 quantize #6e38ff3ffff3fcff 1 -> #6e38ff3fcff3fcff
288 ddcan402 quantize #6e38ff3fcff3fdff 0 -> #6e38ff3fcff3fcff
289 ddcan403 quantize #7880000000000000 Inf -> #7800000000000000
290 ddcan404 quantize #7802000000000000 -Inf -> #7800000000000000
291 ddcan410 quantize #7c03ff3fcff3fcff 1 -> #7c00ff3fcff3fcff
292 ddcan411 quantize #7c03ff3fcff3fcff 1 -> #7c00ff3fcff3fcff
293 ddcan412 quantize #7c40ff3fcff3fcff 1 -> #7c00ff3fcff3fcff
294 ddcan413 quantize #7c40ff3fcff3fcff 1 -> #7c00ff3fcff3fcff
295 ddcan414 quantize #7e00ffffcff3fcff 1 -> #7c00ff3fcff3fcff Invalid_operation
296 ddcan415 quantize #7e00ffffcff3fcff 1 -> #7c00ff3fcff3fcff Invalid_operation
[all …]
|
| D | dqCanonical.decTest | 310 dqcan401 quantize #ee080ff3fcff3fcff3fffffffff3fcff 0 -> #ee080ff3fcff3fcff3fcff3fcff3fcff
311 dqcan402 quantize #ee080ff3fffffffffffcff3fcff3fcff 0 -> #ee080ff3fcff3fcff3fcff3fcff3fcff
312 dqcan403 quantize #78800000000000000000000000000000 Inf -> #78000000000000000000000000000000
313 dqcan404 quantize #78020000000000000000000000000000 -Inf -> #78000000000000000000000000000000
314 dqcan410 quantize #7c080ff3fcff3fcff3fcff3fcff3fcff 1 -> #7c000ff3fcff3fcff3fcff3fcff3fcff
315 dqcan411 quantize #fc000ff3fcfffffff3fcff3fcff3fcff 1 -> #fc000ff3fcff3fcff3fcff3fcff3fcff
316 dqcan412 quantize #7e100ff3fcff3fcff3fcff3fcff3fcff 1 -> #7c000ff3fcff3fcff3fcff3fcff3fcff Inva…
317 dqcan413 quantize #fe000ff3fcff3fcff3ffffffcff3fcff 1 -> #fc000ff3fcff3fcff3fcff3fcff3fcff Inva…
|
| /external/tensorflow/tensorflow/lite/kernels/internal/ |
| D | quantization_util_test.cc | 334 auto quantize = [](double d) { in TEST() local
341 EXPECT_DEATH(quantize(-0.1), ""); in TEST()
342 EXPECT_DEATH(quantize(0.0), ""); in TEST()
343 EXPECT_THAT(quantize(0.25), Pair(1073741824, -1)); in TEST()
347 EXPECT_THAT(quantize(0.50 - 5e-9), Pair(2147483627, -1)); in TEST()
348 EXPECT_THAT(quantize(0.50 - 1e-10), Pair(1073741824, 0)); in TEST()
349 EXPECT_THAT(quantize(0.50), Pair(1073741824, 0)); in TEST()
351 EXPECT_THAT(quantize(0.75), Pair(1610612736, 0)); in TEST()
352 EXPECT_THAT(quantize(1 - 1e-9), Pair(2147483646, 0)); in TEST()
356 EXPECT_DEATH(quantize(1 - 1e-15), ""); in TEST()
[all …]
|
| /external/tensorflow/tensorflow/contrib/quantize/python/ |
| D | quantize_test.py | 23 from tensorflow.contrib.quantize.python import quantize
63 quantize._InsertQuantOp('test', is_training, conv.op, [relu.op],
86 quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
126 quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
168 quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
218 quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
244 quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
276 quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
318 quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
363 quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
[all …]
|
| D | quantize_graph.py | 21 from tensorflow.contrib.quantize.python import fold_batch_norms
22 from tensorflow.contrib.quantize.python import quantize
74 quantize.Quantize(
|
| D | quantize_parameterized_test.py | 22 from tensorflow.contrib.quantize.python import fold_batch_norms
23 from tensorflow.contrib.quantize.python import quantize
205 quantize.Quantize(graph, True, quant_delay=delay)
254 quantize.Quantize(graph, True, quant_delay=delay)
305 quantize.Quantize(graph, True, quant_delay=delay)
356 quantize.Quantize(graph, True, quant_delay=delay)
537 quantize.Quantize(graph, True, quant_delay=delay)
592 quantize.Quantize(graph, True, quant_delay=delay)
650 quantize.Quantize(graph, True, quant_delay=delay)
709 quantize.Quantize(graph, True, quant_delay=delay)
[all …]
|
| /external/libjpeg-turbo/ |
| D | jcdctmgr.c | 44 METHODDEF(void) quantize(JCOEFPTR, DCTELEM *, DCTELEM *);
52 quantize_method_ptr quantize; member
269 fdct->quantize == jsimd_quantize) in start_pass_fdctmgr()
270 fdct->quantize = quantize; in start_pass_fdctmgr()
312 fdct->quantize == jsimd_quantize) in start_pass_fdctmgr()
313 fdct->quantize = quantize; in start_pass_fdctmgr()
408 quantize(JCOEFPTR coef_block, DCTELEM *divisors, DCTELEM *workspace) in quantize() function
504 quantize_method_ptr do_quantize = fdct->quantize; in forward_DCT()
680 fdct->quantize = jsimd_quantize; in jinit_forward_dct()
682 fdct->quantize = quantize; in jinit_forward_dct()
|
| /external/sonivox/jet_tools/JetCreator/ |
| D | JetFile.py | 94 …def __init__ (self, segname, filename, start=None, end=None, length=None, output=None, quantize=No… argument
102 self.quantize = quantize
133 def ExtractEvents (self, start, end, length, quantize, chase_controllers): argument
148 if quantize is not None:
149 quantize = int(quantize)
151 quantize = 0
153 self.Trim(start, end, quantize, chase_controllers=chase_controllers)
331 quantize = config.get(segment_name, SEGMENT_QUANTIZE)
360 …segments.append(JetSegment(segname, filename, start, end, length, output, quantize, jetevents, dls…
439 …jet_segfile.ExtractEvents(segment.start, segment.end, segment.length, segment.quantize, self.confi…
[all …]
|
| /external/tensorflow/tensorflow/lite/kernels/ |
| D | kernel_util.cc | 135 auto quantize = [scale, zero_point](float f) { in CalculateActivationRangeQuantizedImpl() local
140 *act_min = std::max(qmin, quantize(0.0)); in CalculateActivationRangeQuantizedImpl()
143 *act_min = std::max(qmin, quantize(0.0)); in CalculateActivationRangeQuantizedImpl()
144 *act_max = std::min(qmax, quantize(6.0)); in CalculateActivationRangeQuantizedImpl()
146 *act_min = std::max(qmin, quantize(-1.0)); in CalculateActivationRangeQuantizedImpl()
147 *act_max = std::min(qmax, quantize(1.0)); in CalculateActivationRangeQuantizedImpl()
|
| /external/tensorflow/tensorflow/lite/experimental/micro/examples/micro_speech/simple_features/CMSIS/ |
| D | simple_features_generator.cc | 28 void quantize(q15_t* bufA, q15_t* bufB, uint8_t* output);
80 quantize(bufA, bufB, output); in GenerateSimpleFeatures()
85 void quantize(q15_t* bufA, q15_t* bufB, uint8_t* output) { in quantize() function
|
| /external/tensorflow/tensorflow/examples/speech_commands/ |
| D | wav_to_features.py | 48 window_stride_ms, feature_bin_count, quantize, preprocess, argument
88 if quantize:
98 if quantize:
130 FLAGS.feature_bin_count, FLAGS.quantize, FLAGS.preprocess,
|
| /external/tensorflow/tensorflow/contrib/quantize/ |
| D | BUILD | 170 name = "quantize",
171 srcs = ["python/quantize.py"],
190 ":quantize",
215 ":quantize",
237 ":quantize",
|
| /external/tensorflow/tensorflow/contrib/quantization/ |
| D | README.md | 6 the [contrib/quantize](https://www.tensorflow.org/api_docs/python/tf/contrib/quantize)
|
| /external/tensorflow/tensorflow/lite/g3doc/convert/ |
| D | quantization.md | 9 The simplest way to create a small model is to quantize the weights to 8 bits
10 and quantize the inputs/activations "on-the-fly", during inference. This
26 …ntization" nodes](https://github.com/tensorflow/tensorflow/tree/r1.13/tensorflow/contrib/quantize).
|
| /external/mesa3d/src/gallium/drivers/swr/rasterizer/core/ |
| D | depthstencil.h | 101 float quantize = (float)((1 << depthBpc) - 1); in QuantizeDepth() local
102 simdscalar result = _simd_mul_ps(depth, _simd_set1_ps(quantize)); in QuantizeDepth()
108 result = _simd_div_ps(result, _simd_set1_ps(quantize)); in QuantizeDepth()
112 result = _simd_mul_ps(result, _simd_set1_ps(1.0f / quantize)); in QuantizeDepth()
|
| /external/libaom/libaom/av1/encoder/x86/ |
| D | av1_highbd_quantize_avx2.c | 48 static INLINE void quantize(const __m256i *qp, __m256i *c, in quantize() function
106 quantize(qp, &coeff, iscan, log_scale, qcoeff_ptr, dqcoeff_ptr, &eob); in av1_highbd_quantize_fp_avx2()
117 quantize(qp, &coeff, iscan, log_scale, qcoeff_ptr, dqcoeff_ptr, &eob); in av1_highbd_quantize_fp_avx2()
|
| /external/libaom/libaom/aom_dsp/x86/ |
| D | highbd_quantize_intrin_avx2.c | 66 static INLINE void quantize(const __m256i *qp, __m256i *c, in quantize() function
128 quantize(qp, &coeff, iscan, qcoeff_ptr, dqcoeff_ptr, &eob); in aom_highbd_quantize_b_avx2()
140 quantize(qp, &coeff, iscan, qcoeff_ptr, dqcoeff_ptr, &eob); in aom_highbd_quantize_b_avx2()
|